mulmoclaude 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mulmoclaude.js +1 -1
- package/client/assets/{index-KNLBjwuh.css → index-Bm70FDU2.css} +1 -1
- package/client/assets/{index-D8rhwXLq.js → index-eHWB79u5.js} +3 -3
- package/client/index.html +2 -2
- package/package.json +1 -1
- package/server/agent/config.ts +12 -12
- package/server/agent/mcp-server.ts +19 -19
- package/server/agent/mcp-tools/x.ts +5 -5
- package/server/agent/prompt.ts +9 -4
- package/server/agent/sandboxMounts.ts +7 -7
- package/server/agent/stream.ts +4 -4
- package/server/api/routes/files.ts +9 -9
- package/server/api/routes/scheduler.ts +8 -8
- package/server/api/routes/schedulerHandlers.ts +12 -12
- package/server/api/routes/schedulerTasks.ts +14 -14
- package/server/api/routes/sessions.ts +24 -24
- package/server/api/routes/todosColumnsHandlers.ts +30 -30
- package/server/api/routes/wiki.ts +14 -14
- package/server/events/scheduler-adapter.ts +20 -20
- package/server/events/session-store/index.ts +10 -10
- package/server/events/task-manager/index.ts +7 -7
- package/server/index.ts +19 -19
- package/server/utils/date.ts +18 -18
- package/server/utils/files/atomic.ts +9 -9
- package/server/utils/files/html-io.ts +5 -5
- package/server/utils/files/image-store.ts +2 -2
- package/server/utils/files/journal-io.ts +2 -2
- package/server/utils/files/naming.ts +2 -2
- package/server/utils/files/roles-io.ts +10 -10
- package/server/utils/files/scheduler-io.ts +5 -5
- package/server/utils/files/session-io.ts +35 -35
- package/server/utils/files/spreadsheet-store.ts +2 -2
- package/server/utils/files/todos-io.ts +9 -9
- package/server/utils/files/user-tasks-io.ts +5 -5
- package/server/workspace/chat-index/indexer.ts +15 -15
- package/server/workspace/custom-dirs.ts +11 -11
- package/server/workspace/journal/archivist.ts +35 -35
- package/server/workspace/journal/dailyPass.ts +31 -28
- package/server/workspace/journal/indexFile.ts +29 -25
- package/server/workspace/reference-dirs.ts +18 -18
- package/server/workspace/roles.ts +6 -6
- package/server/workspace/skills/discovery.ts +4 -4
- package/server/workspace/skills/user-tasks.ts +34 -34
- package/server/workspace/sources/arxivDiscovery.ts +8 -8
- package/server/workspace/sources/classifier.ts +7 -7
- package/server/workspace/sources/fetchers/arxiv.ts +7 -7
- package/server/workspace/sources/fetchers/githubIssues.ts +7 -7
- package/server/workspace/sources/fetchers/githubReleases.ts +7 -7
- package/server/workspace/sources/interests.ts +9 -9
- package/server/workspace/sources/pipeline/index.ts +6 -6
- package/server/workspace/sources/pipeline/plan.ts +5 -5
- package/server/workspace/sources/registry.ts +16 -16
- package/server/workspace/sources/robots.ts +14 -14
- package/server/workspace/sources/sourceState.ts +11 -9
- package/server/workspace/tool-trace/index.ts +1 -1
- package/server/workspace/tool-trace/writeSearch.ts +26 -16
- package/server/workspace/wiki-backlinks/index.ts +8 -8
- package/server/workspace/wiki-backlinks/sessionBacklinks.ts +15 -15
- package/src/App.vue +30 -30
- package/src/components/ChatInput.vue +7 -7
- package/src/components/LockStatusPopup.vue +2 -2
- package/src/components/NotificationToast.vue +2 -2
- package/src/components/RoleSelector.vue +2 -2
- package/src/components/SessionHistoryPanel.vue +6 -6
- package/src/components/SettingsMcpTab.vue +7 -7
- package/src/components/SettingsModal.vue +3 -3
- package/src/components/SettingsReferenceDirsTab.vue +10 -10
- package/src/components/SettingsWorkspaceDirsTab.vue +5 -5
- package/src/components/SuggestionsPanel.vue +2 -2
- package/src/components/todo/TodoAddDialog.vue +2 -2
- package/src/components/todo/TodoEditPanel.vue +2 -2
- package/src/components/todo/TodoListView.vue +5 -5
- package/src/composables/useCanvasViewMode.ts +5 -5
- package/src/composables/useClickOutside.ts +2 -2
- package/src/composables/useFreshPluginData.ts +3 -3
- package/src/composables/useKeyNavigation.ts +11 -11
- package/src/composables/useMcpTools.ts +2 -2
- package/src/composables/useNotifications.ts +3 -3
- package/src/composables/usePdfDownload.ts +4 -4
- package/src/composables/usePendingCalls.ts +1 -1
- package/src/composables/usePubSub.ts +10 -10
- package/src/composables/useRoles.ts +1 -1
- package/src/composables/useSandboxStatus.ts +1 -1
- package/src/composables/useSessionDerived.ts +3 -3
- package/src/composables/useSessionSync.ts +8 -8
- package/src/composables/useViewLayout.ts +2 -2
- package/src/config/roles.ts +2 -2
- package/src/plugins/chart/Preview.vue +4 -4
- package/src/plugins/manageSkills/View.vue +3 -3
- package/src/plugins/manageSource/Preview.vue +1 -1
- package/src/plugins/markdown/View.vue +2 -2
- package/src/plugins/presentHtml/helpers.ts +8 -8
- package/src/plugins/presentMulmoScript/View.vue +4 -4
- package/src/plugins/presentMulmoScript/helpers.ts +1 -1
- package/src/plugins/scheduler/Preview.vue +6 -6
- package/src/plugins/scheduler/TasksTab.vue +4 -4
- package/src/plugins/textResponse/View.vue +2 -2
- package/src/plugins/todo/Preview.vue +2 -2
- package/src/plugins/todo/View.vue +11 -11
- package/src/plugins/todo/composables/useTodos.ts +5 -5
- package/src/plugins/wiki/Preview.vue +5 -5
- package/src/plugins/wiki/helpers.ts +4 -4
- package/src/router/guards.ts +12 -12
- package/src/types/session.ts +4 -3
- package/src/utils/agent/request.ts +3 -3
- package/src/utils/dom/scrollable.ts +2 -2
- package/src/utils/files/expandedDirs.ts +1 -1
- package/src/utils/files/sortChildren.ts +6 -6
- package/src/utils/format/frontmatter.ts +6 -6
- package/src/utils/image/rewriteMarkdownImageRefs.ts +5 -5
- package/src/utils/markdown/extractFirstH1.ts +2 -2
- package/src/utils/path/relativeLink.ts +15 -15
- package/src/utils/role/icon.ts +2 -2
- package/src/utils/role/merge.ts +2 -2
- package/src/utils/role/plugins.ts +1 -1
- package/src/utils/session/sessionFactory.ts +2 -2
- package/src/utils/session/sessionHelpers.ts +2 -2
- package/src/utils/tools/dedup.ts +4 -4
- package/src/utils/tools/result.ts +3 -3
- package/src/utils/types.ts +2 -2
|
@@ -33,8 +33,8 @@ const DEFAULT_MAX_ITEMS = 20;
|
|
|
33
33
|
* Example: ["transformer", "attention"] → 'ti:"transformer" OR abs:"transformer" OR ti:"attention" OR abs:"attention"'
|
|
34
34
|
*/
|
|
35
35
|
export function buildArxivQuery(keywords: readonly string[]): string {
|
|
36
|
-
const terms = keywords.flatMap((
|
|
37
|
-
const stripped =
|
|
36
|
+
const terms = keywords.flatMap((keyword) => {
|
|
37
|
+
const stripped = keyword.replace(/"/g, "");
|
|
38
38
|
return [`ti:"${stripped}"`, `abs:"${stripped}"`];
|
|
39
39
|
});
|
|
40
40
|
return terms.join(" OR ");
|
|
@@ -48,8 +48,8 @@ export function buildArxivQuery(keywords: readonly string[]): string {
|
|
|
48
48
|
*/
|
|
49
49
|
export function keywordsToSlug(keywords: readonly string[]): string {
|
|
50
50
|
const latin = keywords
|
|
51
|
-
.map((
|
|
52
|
-
.filter((
|
|
51
|
+
.map((keyword) => slugify(keyword, ""))
|
|
52
|
+
.filter((slugPart) => slugPart.length > 0)
|
|
53
53
|
.slice(0, 3)
|
|
54
54
|
.join("-");
|
|
55
55
|
// Short hash of ALL keywords ensures uniqueness even when the
|
|
@@ -91,7 +91,7 @@ export async function discoverAndRegister(root?: string): Promise<DiscoveryResul
|
|
|
91
91
|
fs.mkdirSync(dir, { recursive: true });
|
|
92
92
|
|
|
93
93
|
const existing = await listSources(base);
|
|
94
|
-
const existingSlugs = new Set(existing.map((
|
|
94
|
+
const existingSlugs = new Set(existing.map((source) => source.slug));
|
|
95
95
|
|
|
96
96
|
const registered: string[] = [];
|
|
97
97
|
const skipped: string[] = [];
|
|
@@ -158,17 +158,17 @@ export async function pruneStaleAutoSources(root?: string): Promise<string[]> {
|
|
|
158
158
|
const base = root ?? workspacePath;
|
|
159
159
|
const profile = loadInterests(base);
|
|
160
160
|
const existing = await listSources(base);
|
|
161
|
-
const autoSources = existing.filter((
|
|
161
|
+
const autoSources = existing.filter((source) => source.slug.startsWith(ARXIV_SLUG_PREFIX));
|
|
162
162
|
|
|
163
163
|
if (autoSources.length === 0) return [];
|
|
164
164
|
|
|
165
165
|
// If no profile at all, prune everything auto-registered
|
|
166
|
-
const currentKeywords = profile ? new Set(profile.keywords.map((
|
|
166
|
+
const currentKeywords = profile ? new Set(profile.keywords.map((keyword) => keyword.toLowerCase())) : new Set<string>();
|
|
167
167
|
|
|
168
168
|
const pruned: string[] = [];
|
|
169
169
|
for (const source of autoSources) {
|
|
170
170
|
const notes = (source.notes ?? "").toLowerCase();
|
|
171
|
-
const hasMatch = [...currentKeywords].some((
|
|
171
|
+
const hasMatch = [...currentKeywords].some((keyword) => notes.includes(keyword));
|
|
172
172
|
if (!hasMatch && currentKeywords.size > 0) {
|
|
173
173
|
// Keywords changed — this source is stale but don't delete,
|
|
174
174
|
// just log. User can manually remove via manageSource.
|
|
@@ -115,18 +115,18 @@ export function buildClassifyPrompt(input: ClassifyInput): string {
|
|
|
115
115
|
if (titles.length > 0) {
|
|
116
116
|
lines.push("");
|
|
117
117
|
lines.push("RECENT ITEM TITLES:");
|
|
118
|
-
for (const
|
|
119
|
-
lines.push(`- ${
|
|
118
|
+
for (const title of titles.slice(0, 5)) {
|
|
119
|
+
lines.push(`- ${title}`);
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
122
|
const summaries = input.sampleSummaries ?? [];
|
|
123
123
|
if (summaries.length > 0) {
|
|
124
124
|
lines.push("");
|
|
125
125
|
lines.push("RECENT ITEM SUMMARIES:");
|
|
126
|
-
for (const
|
|
126
|
+
for (const summary of summaries.slice(0, 3)) {
|
|
127
127
|
// One-line truncation so a single long abstract doesn't
|
|
128
128
|
// dominate the prompt budget.
|
|
129
|
-
lines.push(`- ${
|
|
129
|
+
lines.push(`- ${summary.replace(/\s+/g, " ").slice(0, 200)}`);
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
132
|
return lines.join("\n");
|
|
@@ -167,8 +167,8 @@ export function validateClassifyResult(obj: unknown): ClassifyResult {
|
|
|
167
167
|
if (!isRecord(obj)) {
|
|
168
168
|
throw new Error("[sources/classifier] output is not an object");
|
|
169
169
|
}
|
|
170
|
-
const
|
|
171
|
-
const categories = normalizeCategories(
|
|
170
|
+
const record = obj as Record<string, unknown>;
|
|
171
|
+
const categories = normalizeCategories(record.categories);
|
|
172
172
|
if (categories.length === 0) {
|
|
173
173
|
// The model is required to pick at least one (min_items=1 in
|
|
174
174
|
// the schema). If we end up here, something went wrong upstream
|
|
@@ -178,7 +178,7 @@ export function validateClassifyResult(obj: unknown): ClassifyResult {
|
|
|
178
178
|
// with no categories.
|
|
179
179
|
throw new Error("[sources/classifier] output has no valid categories from the taxonomy");
|
|
180
180
|
}
|
|
181
|
-
const rationale = typeof
|
|
181
|
+
const rationale = typeof record.rationale === "string" ? record.rationale.slice(0, 400) : "";
|
|
182
182
|
return { categories, rationale };
|
|
183
183
|
}
|
|
184
184
|
|
|
@@ -90,8 +90,8 @@ export function normalizeArxivFeed(feed: ParsedFeed, source: Source, cursor: Rec
|
|
|
90
90
|
function parseCursorTs(cursor: Record<string, string>): number | null {
|
|
91
91
|
const raw = cursor[ARXIV_CURSOR_KEY];
|
|
92
92
|
if (!raw) return null;
|
|
93
|
-
const
|
|
94
|
-
return Number.isFinite(
|
|
93
|
+
const parsed = Date.parse(raw);
|
|
94
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
// Decide whether one ParsedFeedItem produces a SourceItem given
|
|
@@ -104,8 +104,8 @@ function feedItemToSourceItem(entry: ParsedFeed["items"][number], source: Source
|
|
|
104
104
|
const normalizedUrl = normalizeUrl(entry.link);
|
|
105
105
|
if (!normalizedUrl) return null;
|
|
106
106
|
if (entry.publishedAt && lastSeenTs !== null) {
|
|
107
|
-
const
|
|
108
|
-
if (Number.isFinite(
|
|
107
|
+
const publishedMs = Date.parse(entry.publishedAt);
|
|
108
|
+
if (Number.isFinite(publishedMs) && publishedMs <= lastSeenTs) return null;
|
|
109
109
|
}
|
|
110
110
|
const publishedAt = entry.publishedAt ?? new Date().toISOString();
|
|
111
111
|
return {
|
|
@@ -128,9 +128,9 @@ export function updateArxivCursor(current: Record<string, string>, feed: ParsedF
|
|
|
128
128
|
let newest: number | null = null;
|
|
129
129
|
for (const entry of feed.items) {
|
|
130
130
|
if (!entry.publishedAt) continue;
|
|
131
|
-
const
|
|
132
|
-
if (!Number.isFinite(
|
|
133
|
-
if (newest === null ||
|
|
131
|
+
const publishedMs = Date.parse(entry.publishedAt);
|
|
132
|
+
if (!Number.isFinite(publishedMs)) continue;
|
|
133
|
+
if (newest === null || publishedMs > newest) newest = publishedMs;
|
|
134
134
|
}
|
|
135
135
|
if (newest === null) return current;
|
|
136
136
|
const currentTs = current[ARXIV_CURSOR_KEY] ? Date.parse(current[ARXIV_CURSOR_KEY]) : -Infinity;
|
|
@@ -69,7 +69,7 @@ interface ParsedIssue {
|
|
|
69
69
|
// signal.
|
|
70
70
|
export function parseGithubIssue(raw: unknown): ParsedIssue | null {
|
|
71
71
|
if (!isRecord(raw)) return null;
|
|
72
|
-
const
|
|
72
|
+
const issueId = typeof raw.id === "number" && Number.isFinite(raw.id) ? raw.id : null;
|
|
73
73
|
const issueNumber = typeof raw.number === "number" && Number.isFinite(raw.number) ? raw.number : null;
|
|
74
74
|
const title = typeof raw.title === "string" ? raw.title : null;
|
|
75
75
|
const htmlUrl = typeof raw.html_url === "string" ? raw.html_url : null;
|
|
@@ -81,7 +81,7 @@ export function parseGithubIssue(raw: unknown): ParsedIssue | null {
|
|
|
81
81
|
// object) means this is a PR. Absence means it's an issue.
|
|
82
82
|
const isPr = "pull_request" in raw && raw.pull_request !== undefined && raw.pull_request !== null;
|
|
83
83
|
return {
|
|
84
|
-
id,
|
|
84
|
+
id: issueId,
|
|
85
85
|
number: issueNumber,
|
|
86
86
|
title,
|
|
87
87
|
htmlUrl,
|
|
@@ -110,7 +110,7 @@ export function issueToSourceItem(issue: ParsedIssue, source: Source, params: Is
|
|
|
110
110
|
|
|
111
111
|
const normalizedUrl = normalizeUrl(issue.htmlUrl);
|
|
112
112
|
if (!normalizedUrl) return null;
|
|
113
|
-
const
|
|
113
|
+
const itemId = stableItemId(normalizedUrl);
|
|
114
114
|
|
|
115
115
|
// Title annotations: `[PR]` for pulls, `[closed]` for closed
|
|
116
116
|
// state so the daily summary makes state visible at a glance.
|
|
@@ -123,7 +123,7 @@ export function issueToSourceItem(issue: ParsedIssue, source: Source, params: Is
|
|
|
123
123
|
const summary = issue.body ? firstParagraph(issue.body) : null;
|
|
124
124
|
|
|
125
125
|
return {
|
|
126
|
-
id,
|
|
126
|
+
id: itemId,
|
|
127
127
|
title,
|
|
128
128
|
url: normalizedUrl,
|
|
129
129
|
publishedAt: new Date(updatedTs).toISOString(),
|
|
@@ -139,9 +139,9 @@ export function updateIssuesCursor(current: Record<string, string>, issues: read
|
|
|
139
139
|
for (const issue of issues) {
|
|
140
140
|
if (issue.isPr && !params.includePrs) continue;
|
|
141
141
|
if (!issue.updatedAt) continue;
|
|
142
|
-
const
|
|
143
|
-
if (!Number.isFinite(
|
|
144
|
-
if (newest === null ||
|
|
142
|
+
const updatedMs = Date.parse(issue.updatedAt);
|
|
143
|
+
if (!Number.isFinite(updatedMs)) continue;
|
|
144
|
+
if (newest === null || updatedMs > newest) newest = updatedMs;
|
|
145
145
|
}
|
|
146
146
|
if (newest === null) return current;
|
|
147
147
|
const currentTs = current[ISSUES_CURSOR_KEY] ? Date.parse(current[ISSUES_CURSOR_KEY]) : -Infinity;
|
|
@@ -52,7 +52,7 @@ interface ParsedRelease {
|
|
|
52
52
|
// hitting the network.
|
|
53
53
|
export function parseGithubRelease(raw: unknown): ParsedRelease | null {
|
|
54
54
|
if (!isRecord(raw)) return null;
|
|
55
|
-
const
|
|
55
|
+
const releaseId = typeof raw.id === "number" && Number.isFinite(raw.id) ? raw.id : null;
|
|
56
56
|
const name = typeof raw.name === "string" ? raw.name : null;
|
|
57
57
|
const tagName = typeof raw.tag_name === "string" ? raw.tag_name : null;
|
|
58
58
|
const htmlUrl = typeof raw.html_url === "string" ? raw.html_url : null;
|
|
@@ -60,7 +60,7 @@ export function parseGithubRelease(raw: unknown): ParsedRelease | null {
|
|
|
60
60
|
const publishedAt = typeof raw.published_at === "string" ? raw.published_at : null;
|
|
61
61
|
const draft = raw.draft === true;
|
|
62
62
|
const prerelease = raw.prerelease === true;
|
|
63
|
-
return { id, name, tagName, htmlUrl, body, publishedAt, draft, prerelease };
|
|
63
|
+
return { id: releaseId, name, tagName, htmlUrl, body, publishedAt, draft, prerelease };
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
// Build a SourceItem from a parsed release + the parent Source.
|
|
@@ -82,7 +82,7 @@ export function releaseToSourceItem(release: ParsedRelease, source: Source, last
|
|
|
82
82
|
|
|
83
83
|
const normalizedUrl = normalizeUrl(release.htmlUrl);
|
|
84
84
|
if (!normalizedUrl) return null;
|
|
85
|
-
const
|
|
85
|
+
const itemId = stableItemId(normalizedUrl);
|
|
86
86
|
|
|
87
87
|
// Title resolution: prefer <name> (release display name), fall
|
|
88
88
|
// back to <tag_name> (e.g. "v1.2.3"). Annotate pre-releases so
|
|
@@ -92,7 +92,7 @@ export function releaseToSourceItem(release: ParsedRelease, source: Source, last
|
|
|
92
92
|
const summary = release.body ? firstParagraph(release.body) : null;
|
|
93
93
|
|
|
94
94
|
return {
|
|
95
|
-
id,
|
|
95
|
+
id: itemId,
|
|
96
96
|
title,
|
|
97
97
|
url: normalizedUrl,
|
|
98
98
|
publishedAt: new Date(publishedTs).toISOString(),
|
|
@@ -128,9 +128,9 @@ export function updateReleasesCursor(current: Record<string, string>, releases:
|
|
|
128
128
|
for (const release of releases) {
|
|
129
129
|
if (release.draft) continue;
|
|
130
130
|
if (!release.publishedAt) continue;
|
|
131
|
-
const
|
|
132
|
-
if (!Number.isFinite(
|
|
133
|
-
if (newest === null ||
|
|
131
|
+
const publishedMs = Date.parse(release.publishedAt);
|
|
132
|
+
if (!Number.isFinite(publishedMs)) continue;
|
|
133
|
+
if (newest === null || publishedMs > newest) newest = publishedMs;
|
|
134
134
|
}
|
|
135
135
|
if (newest === null) return current;
|
|
136
136
|
const currentTs = current[RELEASES_CURSOR_KEY] ? Date.parse(current[RELEASES_CURSOR_KEY]) : -Infinity;
|
|
@@ -58,9 +58,9 @@ function validateInterests(raw: unknown): InterestsProfile | null {
|
|
|
58
58
|
const obj = raw as Record<string, unknown>;
|
|
59
59
|
|
|
60
60
|
// Filter out blank/whitespace-only keywords — "" matches every title
|
|
61
|
-
const keywords = Array.isArray(obj.keywords) ? obj.keywords.filter((
|
|
61
|
+
const keywords = Array.isArray(obj.keywords) ? obj.keywords.filter((keyword): keyword is string => isNonEmptyString(keyword)) : [];
|
|
62
62
|
|
|
63
|
-
const categories = Array.isArray(obj.categories) ? obj.categories.filter((
|
|
63
|
+
const categories = Array.isArray(obj.categories) ? obj.categories.filter((category): category is CategorySlug => isCategorySlug(category)) : [];
|
|
64
64
|
|
|
65
65
|
if (keywords.length === 0 && categories.length === 0) return null;
|
|
66
66
|
|
|
@@ -88,16 +88,16 @@ export function scoreItem(item: SourceItem, profile: InterestsProfile): number {
|
|
|
88
88
|
const titleLower = item.title.toLowerCase();
|
|
89
89
|
const summaryLower = (item.summary ?? "").toLowerCase();
|
|
90
90
|
|
|
91
|
-
for (const
|
|
92
|
-
const
|
|
93
|
-
if (titleLower.includes(
|
|
91
|
+
for (const keyword of profile.keywords) {
|
|
92
|
+
const keywordLower = keyword.toLowerCase();
|
|
93
|
+
if (titleLower.includes(keywordLower)) {
|
|
94
94
|
score += KEYWORD_TITLE_WEIGHT;
|
|
95
|
-
} else if (summaryLower.includes(
|
|
95
|
+
} else if (summaryLower.includes(keywordLower)) {
|
|
96
96
|
score += KEYWORD_SUMMARY_WEIGHT;
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
const hasCategory = item.categories.some((
|
|
100
|
+
const hasCategory = item.categories.some((category) => profile.categories.includes(category));
|
|
101
101
|
if (hasCategory) {
|
|
102
102
|
score += CATEGORY_MATCH_WEIGHT;
|
|
103
103
|
}
|
|
@@ -114,7 +114,7 @@ export function scoreItem(item: SourceItem, profile: InterestsProfile): number {
|
|
|
114
114
|
export function scoreAndFilter(items: readonly SourceItem[], profile: InterestsProfile): ScoredItem[] {
|
|
115
115
|
return items
|
|
116
116
|
.map((item) => ({ item, score: scoreItem(item, profile) }))
|
|
117
|
-
.filter((
|
|
118
|
-
.sort((
|
|
117
|
+
.filter((scoredItem) => scoredItem.score >= profile.minRelevance)
|
|
118
|
+
.sort((leftItem, rightItem) => rightItem.score - leftItem.score)
|
|
119
119
|
.slice(0, profile.maxNotificationsPerRun);
|
|
120
120
|
}
|
|
@@ -92,11 +92,11 @@ export { toLocalIsoDate } from "../../../utils/date.js";
|
|
|
92
92
|
// Convert a wall-clock millis value to the LOCAL year-month
|
|
93
93
|
// key (YYYY-MM) used as the archive fallback for items without
|
|
94
94
|
// a parseable publishedAt.
|
|
95
|
-
export function toLocalYearMonth(
|
|
96
|
-
const
|
|
97
|
-
const
|
|
98
|
-
const
|
|
99
|
-
return `${
|
|
95
|
+
export function toLocalYearMonth(millis: number): string {
|
|
96
|
+
const date = new Date(millis);
|
|
97
|
+
const year = date.getFullYear();
|
|
98
|
+
const month = String(date.getMonth() + 1).padStart(2, "0");
|
|
99
|
+
return `${year}-${month}`;
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
export async function runSourcesPipeline(input: RunPipelineInput): Promise<RunPipelineResult> {
|
|
@@ -125,7 +125,7 @@ export async function runSourcesPipeline(input: RunPipelineInput): Promise<RunPi
|
|
|
125
125
|
const allSources = await listSources(workspaceRoot);
|
|
126
126
|
const statesBySlug = await readManyStates(
|
|
127
127
|
workspaceRoot,
|
|
128
|
-
allSources.map((
|
|
128
|
+
allSources.map((source) => source.slug),
|
|
129
129
|
);
|
|
130
130
|
|
|
131
131
|
// --- 2. Plan ------------------------------------------------------
|
|
@@ -31,8 +31,8 @@ export interface PlanInput {
|
|
|
31
31
|
// Sort key: slug, ascending. Deterministic ordering keeps the
|
|
32
32
|
// daily summary's item sequence stable across runs for the same
|
|
33
33
|
// input, which makes markdown diffs readable.
|
|
34
|
-
function bySlug(
|
|
35
|
-
return
|
|
34
|
+
function bySlug(left: Source, right: Source): number {
|
|
35
|
+
return left.slug < right.slug ? -1 : left.slug > right.slug ? 1 : 0;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
// Returns the subset of sources eligible for this cycle. Pure.
|
|
@@ -60,7 +60,7 @@ export function planEligibleSources(input: PlanInput): Source[] {
|
|
|
60
60
|
function isWithinBackoff(state: SourceState | undefined, nowMs: number): boolean {
|
|
61
61
|
if (!state) return false;
|
|
62
62
|
if (!state.nextAttemptAt) return false;
|
|
63
|
-
const
|
|
64
|
-
if (!Number.isFinite(
|
|
65
|
-
return
|
|
63
|
+
const parsed = Date.parse(state.nextAttemptAt);
|
|
64
|
+
if (!Number.isFinite(parsed)) return false;
|
|
65
|
+
return parsed > nowMs;
|
|
66
66
|
}
|
|
@@ -100,43 +100,43 @@ function parseValue(raw: string): string | string[] {
|
|
|
100
100
|
if (arrayMatch) {
|
|
101
101
|
return arrayMatch[1]
|
|
102
102
|
.split(",")
|
|
103
|
-
.map((
|
|
104
|
-
.filter((
|
|
103
|
+
.map((segment) => unquote(segment.trim()))
|
|
104
|
+
.filter((segment) => segment.length > 0);
|
|
105
105
|
}
|
|
106
106
|
return unquote(raw);
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
function unquote(
|
|
109
|
+
function unquote(input: string): string {
|
|
110
110
|
// Double-quoted strings: yamlScalar writes JSON-compatible escape
|
|
111
111
|
// sequences (\\ for \, \" for "), so JSON.parse reverses them in
|
|
112
112
|
// one shot. Fall back to a plain strip if the string is
|
|
113
113
|
// double-quoted but somehow malformed.
|
|
114
|
-
if (
|
|
114
|
+
if (input.length >= 2 && input.startsWith('"') && input.endsWith('"')) {
|
|
115
115
|
try {
|
|
116
|
-
return JSON.parse(
|
|
116
|
+
return JSON.parse(input);
|
|
117
117
|
} catch {
|
|
118
|
-
return
|
|
118
|
+
return input.slice(1, -1);
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
121
|
// Single-quoted scalars follow YAML's doubling convention: '' → '.
|
|
122
|
-
if (
|
|
123
|
-
return
|
|
122
|
+
if (input.length >= 2 && input.startsWith("'") && input.endsWith("'")) {
|
|
123
|
+
return input.slice(1, -1).replace(/''/g, "'");
|
|
124
124
|
}
|
|
125
|
-
return
|
|
125
|
+
return input;
|
|
126
126
|
}
|
|
127
127
|
|
|
128
128
|
// --- Source validation / construction -----------------------------------
|
|
129
129
|
|
|
130
130
|
function stringField(fields: Map<string, string | string[]>, key: string): string | null {
|
|
131
|
-
const
|
|
132
|
-
return isNonEmptyString(
|
|
131
|
+
const value = fields.get(key);
|
|
132
|
+
return isNonEmptyString(value) ? value : null;
|
|
133
133
|
}
|
|
134
134
|
|
|
135
135
|
function numberField(fields: Map<string, string | string[]>, key: string, defaultValue: number): number {
|
|
136
|
-
const
|
|
137
|
-
if (typeof
|
|
138
|
-
const
|
|
139
|
-
return Number.isFinite(
|
|
136
|
+
const value = fields.get(key);
|
|
137
|
+
if (typeof value !== "string") return defaultValue;
|
|
138
|
+
const parsedNumber = Number(value);
|
|
139
|
+
return Number.isFinite(parsedNumber) && parsedNumber > 0 ? Math.floor(parsedNumber) : defaultValue;
|
|
140
140
|
}
|
|
141
141
|
|
|
142
142
|
// Default per-fetch cap. Fetchers treat it as a hint — if the
|
|
@@ -302,7 +302,7 @@ export async function listSources(workspaceRoot: string): Promise<Source[]> {
|
|
|
302
302
|
}
|
|
303
303
|
}
|
|
304
304
|
// Deterministic sort by slug so callers can rely on stable order.
|
|
305
|
-
out.sort((
|
|
305
|
+
out.sort((leftSource, rightSource) => (leftSource.slug < rightSource.slug ? -1 : leftSource.slug > rightSource.slug ? 1 : 0));
|
|
306
306
|
return out;
|
|
307
307
|
}
|
|
308
308
|
|
|
@@ -105,8 +105,8 @@ function applyRule(group: RobotsGroup, name: string, value: string): void {
|
|
|
105
105
|
} else if (name === "allow") {
|
|
106
106
|
group.rules.push({ kind: "allow", pattern: value });
|
|
107
107
|
} else if (name === "crawl-delay") {
|
|
108
|
-
const
|
|
109
|
-
if (Number.isFinite(
|
|
108
|
+
const seconds = Number(value);
|
|
109
|
+
if (Number.isFinite(seconds) && seconds >= 0) group.crawlDelaySec = seconds;
|
|
110
110
|
}
|
|
111
111
|
// Any other directive: ignored.
|
|
112
112
|
}
|
|
@@ -138,13 +138,13 @@ function parseDirective(line: string): { name: string; value: string } | null {
|
|
|
138
138
|
// duplicate group get ignored, which could silently let a fetcher
|
|
139
139
|
// hit a path the site explicitly blocked.
|
|
140
140
|
export function selectGroup(robots: ParsedRobots, userAgent: string): RobotsGroup | null {
|
|
141
|
-
const
|
|
141
|
+
const agent = userAgent.toLowerCase();
|
|
142
142
|
const exacts: RobotsGroup[] = [];
|
|
143
143
|
const stars: RobotsGroup[] = [];
|
|
144
144
|
let bestPrefixScore = -1;
|
|
145
145
|
let prefixMatches: RobotsGroup[] = [];
|
|
146
146
|
for (const group of robots.groups) {
|
|
147
|
-
const outcome = scoreGroupAgainstAgent(group,
|
|
147
|
+
const outcome = scoreGroupAgainstAgent(group, agent);
|
|
148
148
|
if (outcome.kind === "exact") exacts.push(outcome.group);
|
|
149
149
|
else if (outcome.kind === "star") stars.push(outcome.group);
|
|
150
150
|
else if (outcome.kind === "prefix") {
|
|
@@ -170,11 +170,11 @@ function mergeGroups(groups: readonly RobotsGroup[]): RobotsGroup {
|
|
|
170
170
|
const rules: RobotsRule[] = [];
|
|
171
171
|
const userAgents: string[] = [];
|
|
172
172
|
let crawlDelaySec: number | null = null;
|
|
173
|
-
for (const
|
|
174
|
-
rules.push(...
|
|
175
|
-
userAgents.push(...
|
|
176
|
-
if (
|
|
177
|
-
crawlDelaySec = crawlDelaySec === null ?
|
|
173
|
+
for (const group of groups) {
|
|
174
|
+
rules.push(...group.rules);
|
|
175
|
+
userAgents.push(...group.userAgents);
|
|
176
|
+
if (group.crawlDelaySec !== null) {
|
|
177
|
+
crawlDelaySec = crawlDelaySec === null ? group.crawlDelaySec : Math.min(crawlDelaySec, group.crawlDelaySec);
|
|
178
178
|
}
|
|
179
179
|
}
|
|
180
180
|
return { userAgents, rules, crawlDelaySec };
|
|
@@ -186,7 +186,7 @@ type AgentMatch =
|
|
|
186
186
|
| { kind: "star"; group: RobotsGroup }
|
|
187
187
|
| { kind: "none" };
|
|
188
188
|
|
|
189
|
-
function scoreGroupAgainstAgent(group: RobotsGroup,
|
|
189
|
+
function scoreGroupAgainstAgent(group: RobotsGroup, agent: string): AgentMatch {
|
|
190
190
|
let bestPrefix = -1;
|
|
191
191
|
let hasStar = false;
|
|
192
192
|
for (const listed of group.userAgents) {
|
|
@@ -194,8 +194,8 @@ function scoreGroupAgainstAgent(group: RobotsGroup, ua: string): AgentMatch {
|
|
|
194
194
|
hasStar = true;
|
|
195
195
|
continue;
|
|
196
196
|
}
|
|
197
|
-
if (listed ===
|
|
198
|
-
if (
|
|
197
|
+
if (listed === agent) return { kind: "exact", group };
|
|
198
|
+
if (agent.startsWith(listed) && listed.length > bestPrefix) {
|
|
199
199
|
bestPrefix = listed.length;
|
|
200
200
|
}
|
|
201
201
|
}
|
|
@@ -266,6 +266,6 @@ export function matchesPattern(pattern: string, path: string): number {
|
|
|
266
266
|
.split("*")
|
|
267
267
|
.map((chunk) => chunk.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
|
|
268
268
|
.join(".*");
|
|
269
|
-
const
|
|
270
|
-
return
|
|
269
|
+
const regex = new RegExp("^" + regexBody + (endAnchored ? "$" : ""));
|
|
270
|
+
return regex.test(path) ? pattern.length : -1;
|
|
271
271
|
}
|
|
@@ -47,12 +47,14 @@ export function validateSourceState(raw: unknown, slug: string): SourceState {
|
|
|
47
47
|
if (!isRecord(raw)) {
|
|
48
48
|
return defaultSourceState(slug);
|
|
49
49
|
}
|
|
50
|
-
const
|
|
51
|
-
const lastFetchedAt = typeof
|
|
52
|
-
const nextAttemptAt = typeof
|
|
50
|
+
const record = raw as Record<string, unknown>;
|
|
51
|
+
const lastFetchedAt = typeof record.lastFetchedAt === "string" ? record.lastFetchedAt : null;
|
|
52
|
+
const nextAttemptAt = typeof record.nextAttemptAt === "string" ? record.nextAttemptAt : null;
|
|
53
53
|
const consecutiveFailures =
|
|
54
|
-
typeof
|
|
55
|
-
|
|
54
|
+
typeof record.consecutiveFailures === "number" && Number.isFinite(record.consecutiveFailures) && record.consecutiveFailures >= 0
|
|
55
|
+
? Math.floor(record.consecutiveFailures)
|
|
56
|
+
: 0;
|
|
57
|
+
const cursor = validateCursor(record.cursor);
|
|
56
58
|
return {
|
|
57
59
|
slug,
|
|
58
60
|
lastFetchedAt,
|
|
@@ -127,9 +129,9 @@ export async function deleteSourceState(workspaceRoot: string, slug: string): Pr
|
|
|
127
129
|
// deterministic regardless of which fetcher finished first.
|
|
128
130
|
// Used by reporting / logging code.
|
|
129
131
|
export function sortBySlug<T extends { sourceSlug?: string; slug?: string }>(items: readonly T[]): T[] {
|
|
130
|
-
return [...items].sort((
|
|
131
|
-
const
|
|
132
|
-
const
|
|
133
|
-
return
|
|
132
|
+
return [...items].sort((left, right) => {
|
|
133
|
+
const leftKey = left.sourceSlug ?? left.slug ?? "";
|
|
134
|
+
const rightKey = right.sourceSlug ?? right.slug ?? "";
|
|
135
|
+
return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0;
|
|
134
136
|
});
|
|
135
137
|
}
|
|
@@ -225,7 +225,7 @@ async function maybeWriteSearch(inputs: MaybeWriteSearchInputs): Promise<string
|
|
|
225
225
|
workspaceRoot: inputs.workspaceRoot,
|
|
226
226
|
query,
|
|
227
227
|
sessionId: inputs.chatSessionId,
|
|
228
|
-
|
|
228
|
+
timestamp: inputs.now,
|
|
229
229
|
resultBody: inputs.content,
|
|
230
230
|
});
|
|
231
231
|
} catch (err) {
|
|
@@ -21,29 +21,29 @@ const MAX_QUERY_SLUG_CHARS = 40;
|
|
|
21
21
|
// from this module. The function is now in utils/date.ts.
|
|
22
22
|
export { toUtcIsoDate as formatSearchDateDir } from "../../utils/date.js";
|
|
23
23
|
|
|
24
|
-
export function computeSearchHash(query: string, sessionId: string,
|
|
25
|
-
return createHash("sha256").update(`${query}\n${sessionId}\n${
|
|
24
|
+
export function computeSearchHash(query: string, sessionId: string, timestamp: Date): string {
|
|
25
|
+
return createHash("sha256").update(`${query}\n${sessionId}\n${timestamp.toISOString()}`, "utf-8").digest("base64url").slice(0, SEARCH_HASH_LEN);
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
export interface SearchPathInputs {
|
|
29
29
|
query: string;
|
|
30
30
|
sessionId: string;
|
|
31
|
-
|
|
31
|
+
timestamp: Date;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
// Returns the workspace-relative path (POSIX slashes) where the search
|
|
35
35
|
// file should live, e.g. "conversations/searches/2026-04-13/foo-abc12345.md".
|
|
36
36
|
export function computeSearchRelPath(inputs: SearchPathInputs): string {
|
|
37
37
|
const slug = slugify(inputs.query, "search", MAX_QUERY_SLUG_CHARS);
|
|
38
|
-
const hash = computeSearchHash(inputs.query, inputs.sessionId, inputs.
|
|
39
|
-
const dateDir = toUtcIsoDate(inputs.
|
|
38
|
+
const hash = computeSearchHash(inputs.query, inputs.sessionId, inputs.timestamp);
|
|
39
|
+
const dateDir = toUtcIsoDate(inputs.timestamp);
|
|
40
40
|
return path.posix.join(SEARCHES_DIR, dateDir, `${slug}-${hash}.md`);
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
export interface SearchContentInputs {
|
|
44
44
|
query: string;
|
|
45
45
|
sessionId: string;
|
|
46
|
-
|
|
46
|
+
timestamp: Date;
|
|
47
47
|
resultBody: string;
|
|
48
48
|
}
|
|
49
49
|
|
|
@@ -51,17 +51,27 @@ export interface SearchContentInputs {
|
|
|
51
51
|
// machine-readable metadata, then a human-readable heading, then the
|
|
52
52
|
// raw search result body verbatim.
|
|
53
53
|
export function buildSearchMarkdown(inputs: SearchContentInputs): string {
|
|
54
|
-
const { query, sessionId,
|
|
54
|
+
const { query, sessionId, timestamp, resultBody } = inputs;
|
|
55
55
|
const body = resultBody.endsWith("\n") ? resultBody : `${resultBody}\n`;
|
|
56
|
-
return [
|
|
56
|
+
return [
|
|
57
|
+
"---",
|
|
58
|
+
`query: ${jsonStringSafe(query)}`,
|
|
59
|
+
`sessionId: ${sessionId}`,
|
|
60
|
+
`ts: ${timestamp.toISOString()}`,
|
|
61
|
+
"---",
|
|
62
|
+
"",
|
|
63
|
+
`# Search: ${query}`,
|
|
64
|
+
"",
|
|
65
|
+
body,
|
|
66
|
+
].join("\n");
|
|
57
67
|
}
|
|
58
68
|
|
|
59
69
|
// Quote a string for YAML only when it could otherwise be
|
|
60
70
|
// misinterpreted (contains a colon, hash, or leading/trailing space).
|
|
61
71
|
// Cheap and good enough for a machine-authored frontmatter.
|
|
62
|
-
function jsonStringSafe(
|
|
63
|
-
const needsQuote = /[:#\n]/.test(
|
|
64
|
-
return needsQuote ? JSON.stringify(
|
|
72
|
+
function jsonStringSafe(input: string): string {
|
|
73
|
+
const needsQuote = /[:#\n]/.test(input) || input !== input.trim();
|
|
74
|
+
return needsQuote ? JSON.stringify(input) : input;
|
|
65
75
|
}
|
|
66
76
|
|
|
67
77
|
export interface WriteSearchInputs extends SearchContentInputs {
|
|
@@ -77,7 +87,7 @@ const defaultDeps: WriteSearchDeps = {
|
|
|
77
87
|
mkdir: async (dir) => {
|
|
78
88
|
await fsp.mkdir(dir, { recursive: true });
|
|
79
89
|
},
|
|
80
|
-
writeFile: (
|
|
90
|
+
writeFile: (filePath, content) => fsp.writeFile(filePath, content, "utf-8"),
|
|
81
91
|
};
|
|
82
92
|
|
|
83
93
|
/**
|
|
@@ -85,14 +95,14 @@ const defaultDeps: WriteSearchDeps = {
|
|
|
85
95
|
* path that should be used as the jsonl `contentRef`.
|
|
86
96
|
*/
|
|
87
97
|
export async function writeSearchResult(inputs: WriteSearchInputs, deps: Partial<WriteSearchDeps> = {}): Promise<string> {
|
|
88
|
-
const d: WriteSearchDeps = { ...defaultDeps, ...deps };
|
|
89
98
|
const relPath = computeSearchRelPath({
|
|
90
99
|
query: inputs.query,
|
|
91
100
|
sessionId: inputs.sessionId,
|
|
92
|
-
|
|
101
|
+
timestamp: inputs.timestamp,
|
|
93
102
|
});
|
|
94
103
|
const absPath = path.join(inputs.workspaceRoot, relPath);
|
|
95
|
-
|
|
96
|
-
await
|
|
104
|
+
const activeDeps: WriteSearchDeps = { ...defaultDeps, ...deps };
|
|
105
|
+
await activeDeps.mkdir(path.dirname(absPath));
|
|
106
|
+
await activeDeps.writeFile(absPath, buildSearchMarkdown(inputs));
|
|
97
107
|
return relPath;
|
|
98
108
|
}
|
|
@@ -30,16 +30,16 @@ const MTIME_TOLERANCE_MS = ONE_SECOND_MS;
|
|
|
30
30
|
|
|
31
31
|
export interface WikiBacklinksDeps {
|
|
32
32
|
readdir: (dir: string) => Promise<string[]>;
|
|
33
|
-
stat: (
|
|
34
|
-
readFile: (
|
|
35
|
-
writeFile: (
|
|
33
|
+
stat: (filePath: string) => Promise<{ mtimeMs: number }>;
|
|
34
|
+
readFile: (filePath: string) => Promise<string>;
|
|
35
|
+
writeFile: (filePath: string, content: string) => Promise<void>;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
const defaultDeps: WikiBacklinksDeps = {
|
|
39
39
|
readdir: (dir) => fsp.readdir(dir),
|
|
40
|
-
stat: (
|
|
41
|
-
readFile: (
|
|
42
|
-
writeFile: (
|
|
40
|
+
stat: (filePath) => fsp.stat(filePath),
|
|
41
|
+
readFile: (filePath) => fsp.readFile(filePath, "utf-8"),
|
|
42
|
+
writeFile: (filePath, content) => fsp.writeFile(filePath, content, "utf-8"),
|
|
43
43
|
};
|
|
44
44
|
|
|
45
45
|
export interface MaybeAppendWikiBacklinksOptions {
|
|
@@ -78,8 +78,8 @@ async function listPageFiles(pagesDir: string, deps: WikiBacklinksDeps): Promise
|
|
|
78
78
|
async function processOneFile(pagesDir: string, fileName: string, sessionId: string, mtimeThreshold: number, deps: WikiBacklinksDeps): Promise<void> {
|
|
79
79
|
const fullPath = path.join(pagesDir, fileName);
|
|
80
80
|
try {
|
|
81
|
-
const
|
|
82
|
-
if (
|
|
81
|
+
const stats = await deps.stat(fullPath);
|
|
82
|
+
if (stats.mtimeMs < mtimeThreshold) return;
|
|
83
83
|
|
|
84
84
|
const content = await deps.readFile(fullPath);
|
|
85
85
|
// Compute the relative path from the wiki page's directory to
|