@oh-my-pi/pi-coding-agent 12.18.3 → 12.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/package.json +7 -7
- package/src/async/index.ts +1 -0
- package/src/async/job-manager.ts +341 -0
- package/src/cli/file-processor.ts +3 -3
- package/src/cli/list-models.ts +3 -17
- package/src/cli/stats-cli.ts +3 -22
- package/src/cli/web-search-cli.ts +8 -16
- package/src/commit/agentic/agent.ts +6 -9
- package/src/commit/agentic/index.ts +44 -50
- package/src/commit/agentic/state.ts +0 -9
- package/src/commit/agentic/tools/propose-commit.ts +1 -30
- package/src/commit/agentic/tools/schemas.ts +31 -0
- package/src/commit/agentic/tools/split-commit.ts +1 -30
- package/src/commit/agentic/validation.ts +1 -18
- package/src/commit/analysis/conventional.ts +3 -50
- package/src/commit/analysis/summary.ts +2 -13
- package/src/commit/changelog/detect.ts +4 -1
- package/src/commit/changelog/generate.ts +2 -25
- package/src/commit/changelog/index.ts +1 -2
- package/src/commit/cli.ts +4 -12
- package/src/commit/map-reduce/reduce-phase.ts +2 -43
- package/src/commit/pipeline.ts +7 -15
- package/src/commit/utils.ts +44 -0
- package/src/config/prompt-templates.ts +1 -81
- package/src/config/settings-schema.ts +20 -1
- package/src/config.ts +2 -3
- package/src/debug/index.ts +1 -6
- package/src/debug/system-info.ts +2 -6
- package/src/discovery/builtin.ts +5 -9
- package/src/discovery/helpers.ts +0 -26
- package/src/discovery/ssh.ts +1 -8
- package/src/exa/company.ts +8 -39
- package/src/exa/factory.ts +64 -0
- package/src/exa/index.ts +0 -16
- package/src/exa/linkedin.ts +8 -39
- package/src/exa/mcp-client.ts +0 -64
- package/src/exa/researcher.ts +17 -59
- package/src/exa/search.ts +30 -154
- package/src/extensibility/custom-tools/loader.ts +3 -41
- package/src/extensibility/extensions/loader.ts +2 -9
- package/src/extensibility/hooks/loader.ts +3 -20
- package/src/extensibility/hooks/runner.ts +3 -19
- package/src/extensibility/plugins/installer.ts +2 -1
- package/src/extensibility/plugins/loader.ts +29 -117
- package/src/extensibility/skills.ts +2 -89
- package/src/extensibility/slash-commands.ts +1 -63
- package/src/extensibility/utils.ts +38 -0
- package/src/index.ts +9 -25
- package/src/internal-urls/index.ts +1 -0
- package/src/internal-urls/jobs-protocol.ts +118 -0
- package/src/ipy/kernel.ts +2 -0
- package/src/lsp/config.ts +1 -5
- package/src/lsp/lspmux.ts +0 -17
- package/src/lsp/utils.ts +2 -24
- package/src/main.ts +16 -24
- package/src/mcp/client.ts +1 -46
- package/src/mcp/render.ts +8 -1
- package/src/mcp/tool-cache.ts +1 -5
- package/src/mcp/transports/http.ts +2 -7
- package/src/mcp/transports/stdio.ts +2 -7
- package/src/modes/components/bash-execution.ts +2 -16
- package/src/modes/components/extensions/inspector-panel.ts +8 -18
- package/src/modes/components/footer.ts +10 -50
- package/src/modes/components/model-selector.ts +2 -21
- package/src/modes/components/python-execution.ts +2 -16
- package/src/modes/components/settings-selector.ts +1 -10
- package/src/modes/components/status-line/segments.ts +8 -25
- package/src/modes/components/status-line.ts +14 -31
- package/src/modes/components/tool-execution.ts +8 -2
- package/src/modes/controllers/command-controller.ts +71 -30
- package/src/modes/controllers/event-controller.ts +34 -4
- package/src/modes/controllers/mcp-command-controller.ts +3 -34
- package/src/modes/controllers/selector-controller.ts +2 -2
- package/src/modes/controllers/ssh-command-controller.ts +3 -34
- package/src/modes/interactive-mode.ts +6 -2
- package/src/modes/rpc/rpc-client.ts +1 -5
- package/src/modes/shared.ts +73 -0
- package/src/modes/types.ts +1 -0
- package/src/modes/utils/ui-helpers.ts +26 -2
- package/src/patch/index.ts +4 -4
- package/src/patch/normalize.ts +22 -65
- package/src/patch/shared.ts +16 -16
- package/src/prompts/system/custom-system-prompt.md +0 -10
- package/src/prompts/system/system-prompt.md +69 -89
- package/src/prompts/tools/async-result.md +5 -0
- package/src/prompts/tools/bash.md +5 -0
- package/src/prompts/tools/cancel-job.md +7 -0
- package/src/prompts/tools/poll-jobs.md +7 -0
- package/src/prompts/tools/task.md +4 -0
- package/src/sdk.ts +70 -6
- package/src/session/agent-session.ts +40 -6
- package/src/session/agent-storage.ts +69 -278
- package/src/session/auth-storage.ts +14 -1430
- package/src/session/session-manager.ts +69 -5
- package/src/session/session-storage.ts +1 -5
- package/src/session/streaming-output.ts +637 -76
- package/src/slash-commands/builtin-registry.ts +8 -0
- package/src/ssh/connection-manager.ts +4 -12
- package/src/ssh/sshfs-mount.ts +3 -7
- package/src/ssh/utils.ts +8 -0
- package/src/system-prompt.ts +24 -90
- package/src/task/executor.ts +11 -1
- package/src/task/index.ts +258 -13
- package/src/task/parallel.ts +32 -0
- package/src/task/render.ts +15 -7
- package/src/task/types.ts +5 -0
- package/src/tools/ask.ts +4 -7
- package/src/tools/bash-interactive.ts +4 -5
- package/src/tools/bash.ts +125 -41
- package/src/tools/cancel-job.ts +93 -0
- package/src/tools/fetch.ts +7 -27
- package/src/tools/find.ts +3 -3
- package/src/tools/gemini-image.ts +15 -14
- package/src/tools/grep.ts +3 -3
- package/src/tools/index.ts +13 -29
- package/src/tools/json-tree.ts +12 -1
- package/src/tools/jtd-to-json-schema.ts +10 -74
- package/src/tools/jtd-to-typescript.ts +10 -72
- package/src/tools/jtd-utils.ts +102 -0
- package/src/tools/notebook.ts +4 -9
- package/src/tools/output-meta.ts +52 -26
- package/src/tools/path-utils.ts +13 -7
- package/src/tools/poll-jobs.ts +178 -0
- package/src/tools/python.ts +32 -35
- package/src/tools/read.ts +61 -82
- package/src/tools/render-utils.ts +8 -159
- package/src/tools/ssh.ts +7 -20
- package/src/tools/submit-result.ts +1 -1
- package/src/tools/tool-errors.ts +0 -30
- package/src/tools/tool-result.ts +1 -2
- package/src/tools/write.ts +8 -10
- package/src/tui/code-cell.ts +8 -3
- package/src/tui/status-line.ts +4 -4
- package/src/tui/types.ts +0 -1
- package/src/tui/utils.ts +1 -14
- package/src/utils/command-args.ts +76 -0
- package/src/utils/file-mentions.ts +15 -19
- package/src/utils/frontmatter.ts +5 -10
- package/src/utils/shell-snapshot.ts +0 -11
- package/src/utils/title-generator.ts +0 -12
- package/src/web/scrapers/artifacthub.ts +7 -16
- package/src/web/scrapers/arxiv.ts +3 -8
- package/src/web/scrapers/aur.ts +8 -22
- package/src/web/scrapers/biorxiv.ts +5 -14
- package/src/web/scrapers/bluesky.ts +13 -36
- package/src/web/scrapers/brew.ts +5 -10
- package/src/web/scrapers/cheatsh.ts +2 -12
- package/src/web/scrapers/chocolatey.ts +63 -26
- package/src/web/scrapers/choosealicense.ts +3 -18
- package/src/web/scrapers/cisa-kev.ts +4 -18
- package/src/web/scrapers/clojars.ts +6 -33
- package/src/web/scrapers/coingecko.ts +25 -33
- package/src/web/scrapers/crates-io.ts +7 -26
- package/src/web/scrapers/crossref.ts +4 -18
- package/src/web/scrapers/devto.ts +11 -41
- package/src/web/scrapers/discogs.ts +7 -10
- package/src/web/scrapers/discourse.ts +6 -31
- package/src/web/scrapers/dockerhub.ts +12 -35
- package/src/web/scrapers/fdroid.ts +8 -33
- package/src/web/scrapers/firefox-addons.ts +10 -34
- package/src/web/scrapers/flathub.ts +7 -24
- package/src/web/scrapers/github-gist.ts +2 -12
- package/src/web/scrapers/github.ts +9 -47
- package/src/web/scrapers/gitlab.ts +130 -185
- package/src/web/scrapers/go-pkg.ts +12 -22
- package/src/web/scrapers/hackage.ts +88 -43
- package/src/web/scrapers/hackernews.ts +25 -45
- package/src/web/scrapers/hex.ts +19 -36
- package/src/web/scrapers/huggingface.ts +26 -91
- package/src/web/scrapers/iacr.ts +3 -8
- package/src/web/scrapers/jetbrains-marketplace.ts +9 -20
- package/src/web/scrapers/lemmy.ts +5 -23
- package/src/web/scrapers/lobsters.ts +16 -28
- package/src/web/scrapers/mastodon.ts +24 -43
- package/src/web/scrapers/maven.ts +6 -21
- package/src/web/scrapers/mdn.ts +7 -11
- package/src/web/scrapers/metacpan.ts +9 -41
- package/src/web/scrapers/musicbrainz.ts +4 -28
- package/src/web/scrapers/npm.ts +8 -25
- package/src/web/scrapers/nuget.ts +14 -37
- package/src/web/scrapers/nvd.ts +6 -28
- package/src/web/scrapers/ollama.ts +7 -34
- package/src/web/scrapers/open-vsx.ts +5 -19
- package/src/web/scrapers/opencorporates.ts +30 -14
- package/src/web/scrapers/openlibrary.ts +49 -33
- package/src/web/scrapers/orcid.ts +4 -18
- package/src/web/scrapers/osv.ts +7 -24
- package/src/web/scrapers/packagist.ts +9 -24
- package/src/web/scrapers/pub-dev.ts +7 -50
- package/src/web/scrapers/pubmed.ts +54 -21
- package/src/web/scrapers/pypi.ts +8 -26
- package/src/web/scrapers/rawg.ts +11 -19
- package/src/web/scrapers/readthedocs.ts +4 -9
- package/src/web/scrapers/reddit.ts +5 -15
- package/src/web/scrapers/repology.ts +8 -20
- package/src/web/scrapers/rfc.ts +5 -14
- package/src/web/scrapers/rubygems.ts +6 -21
- package/src/web/scrapers/searchcode.ts +8 -36
- package/src/web/scrapers/sec-edgar.ts +4 -18
- package/src/web/scrapers/semantic-scholar.ts +15 -35
- package/src/web/scrapers/snapcraft.ts +5 -19
- package/src/web/scrapers/sourcegraph.ts +5 -43
- package/src/web/scrapers/spdx.ts +4 -18
- package/src/web/scrapers/spotify.ts +4 -23
- package/src/web/scrapers/stackoverflow.ts +8 -13
- package/src/web/scrapers/terraform.ts +9 -37
- package/src/web/scrapers/tldr.ts +3 -7
- package/src/web/scrapers/twitter.ts +3 -7
- package/src/web/scrapers/types.ts +105 -27
- package/src/web/scrapers/utils.ts +97 -103
- package/src/web/scrapers/vimeo.ts +7 -27
- package/src/web/scrapers/vscode-marketplace.ts +8 -17
- package/src/web/scrapers/w3c.ts +6 -14
- package/src/web/scrapers/wikidata.ts +5 -19
- package/src/web/scrapers/wikipedia.ts +2 -12
- package/src/web/scrapers/youtube.ts +5 -34
- package/src/web/search/index.ts +0 -9
- package/src/web/search/providers/anthropic.ts +3 -2
- package/src/web/search/providers/brave.ts +3 -18
- package/src/web/search/providers/exa.ts +1 -12
- package/src/web/search/providers/kimi.ts +5 -44
- package/src/web/search/providers/perplexity.ts +1 -12
- package/src/web/search/providers/synthetic.ts +3 -26
- package/src/web/search/providers/utils.ts +36 -0
- package/src/web/search/providers/zai.ts +9 -50
- package/src/web/search/types.ts +0 -28
- package/src/web/search/utils.ts +17 -0
- package/src/tools/output-utils.ts +0 -63
- package/src/tools/truncate.ts +0 -385
- package/src/web/search/auth.ts +0 -178
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
// =============================================================================
|
|
5
5
|
// Lobste.rs Types
|
|
@@ -10,9 +10,7 @@ interface LobstersStory {
|
|
|
10
10
|
title: string;
|
|
11
11
|
url?: string;
|
|
12
12
|
description?: string;
|
|
13
|
-
submitter_user:
|
|
14
|
-
username: string;
|
|
15
|
-
};
|
|
13
|
+
submitter_user: string;
|
|
16
14
|
score: number;
|
|
17
15
|
comment_count: number;
|
|
18
16
|
created_at: string;
|
|
@@ -22,9 +20,7 @@ interface LobstersStory {
|
|
|
22
20
|
interface LobstersComment {
|
|
23
21
|
short_id: string;
|
|
24
22
|
comment: string;
|
|
25
|
-
commenting_user:
|
|
26
|
-
username: string;
|
|
27
|
-
};
|
|
23
|
+
commenting_user: string;
|
|
28
24
|
score: number;
|
|
29
25
|
created_at: string;
|
|
30
26
|
indent_level: number;
|
|
@@ -36,9 +32,7 @@ interface LobstersStoryResponse {
|
|
|
36
32
|
title: string;
|
|
37
33
|
url?: string;
|
|
38
34
|
description?: string;
|
|
39
|
-
submitter_user:
|
|
40
|
-
username: string;
|
|
41
|
-
};
|
|
35
|
+
submitter_user: string;
|
|
42
36
|
score: number;
|
|
43
37
|
comment_count: number;
|
|
44
38
|
created_at: string;
|
|
@@ -59,7 +53,7 @@ function renderComments(comments: LobstersComment[], maxDepth = 5): string {
|
|
|
59
53
|
if (comment.indent_level >= maxDepth) continue;
|
|
60
54
|
|
|
61
55
|
const indent = " ".repeat(comment.indent_level);
|
|
62
|
-
md += `${indent}### ${comment.commenting_user
|
|
56
|
+
md += `${indent}### ${comment.commenting_user} · ${comment.score} points\n\n`;
|
|
63
57
|
md += `${indent}${comment.comment.split("\n").join(`\n${indent}`)}\n\n`;
|
|
64
58
|
|
|
65
59
|
if (comment.comments && comment.comments.length > 0) {
|
|
@@ -90,15 +84,16 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
|
|
|
90
84
|
const result = await loadPage(jsonUrl, { timeout, signal });
|
|
91
85
|
if (!result.ok) return null;
|
|
92
86
|
|
|
93
|
-
const story =
|
|
87
|
+
const story = tryParseJson<LobstersStoryResponse>(result.content);
|
|
88
|
+
if (!story) return null;
|
|
94
89
|
|
|
95
90
|
md = `# ${story.title}\n\n`;
|
|
96
|
-
md += `**${story.submitter_user
|
|
91
|
+
md += `**${story.submitter_user}** · ${story.score} points · ${story.comment_count} comments`;
|
|
97
92
|
if (story.tags.length > 0) {
|
|
98
93
|
md += ` · [${story.tags.join(", ")}]`;
|
|
99
94
|
}
|
|
100
95
|
md += `\n`;
|
|
101
|
-
md += `*${
|
|
96
|
+
md += `*${formatIsoDate(story.created_at)}*\n\n`;
|
|
102
97
|
|
|
103
98
|
if (story.description) {
|
|
104
99
|
md += `---\n\n${story.description}\n\n`;
|
|
@@ -112,17 +107,13 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
|
|
|
112
107
|
md += renderComments(story.comments);
|
|
113
108
|
}
|
|
114
109
|
|
|
115
|
-
|
|
116
|
-
return {
|
|
110
|
+
return buildResult(md, {
|
|
117
111
|
url,
|
|
118
112
|
finalUrl: jsonUrl,
|
|
119
|
-
contentType: "text/markdown",
|
|
120
113
|
method: "lobsters",
|
|
121
|
-
content: output.content,
|
|
122
114
|
fetchedAt,
|
|
123
|
-
truncated: output.truncated,
|
|
124
115
|
notes: ["Fetched via Lobste.rs JSON API"],
|
|
125
|
-
};
|
|
116
|
+
});
|
|
126
117
|
}
|
|
127
118
|
|
|
128
119
|
// Front page, newest, or tag page
|
|
@@ -143,7 +134,8 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
|
|
|
143
134
|
const result = await loadPage(jsonUrl, { timeout, signal });
|
|
144
135
|
if (!result.ok) return null;
|
|
145
136
|
|
|
146
|
-
const stories =
|
|
137
|
+
const stories = tryParseJson<LobstersStory[]>(result.content);
|
|
138
|
+
if (!stories) return null;
|
|
147
139
|
const listingStories = stories.slice(0, 20);
|
|
148
140
|
|
|
149
141
|
const title =
|
|
@@ -157,7 +149,7 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
|
|
|
157
149
|
|
|
158
150
|
for (const story of listingStories) {
|
|
159
151
|
md += `- **${story.title}** (${story.score} pts, ${story.comment_count} comments)\n`;
|
|
160
|
-
md += ` by ${story.submitter_user
|
|
152
|
+
md += ` by ${story.submitter_user}`;
|
|
161
153
|
if (story.tags.length > 0) {
|
|
162
154
|
md += ` · [${story.tags.join(", ")}]`;
|
|
163
155
|
}
|
|
@@ -168,17 +160,13 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
|
|
|
168
160
|
md += ` https://lobste.rs/s/${story.short_id}\n\n`;
|
|
169
161
|
}
|
|
170
162
|
|
|
171
|
-
|
|
172
|
-
return {
|
|
163
|
+
return buildResult(md, {
|
|
173
164
|
url,
|
|
174
165
|
finalUrl: jsonUrl,
|
|
175
|
-
contentType: "text/markdown",
|
|
176
166
|
method: "lobsters",
|
|
177
|
-
content: output.content,
|
|
178
167
|
fetchedAt,
|
|
179
|
-
truncated: output.truncated,
|
|
180
168
|
notes: ["Fetched via Lobste.rs JSON API"],
|
|
181
|
-
};
|
|
169
|
+
});
|
|
182
170
|
}
|
|
183
171
|
} catch {}
|
|
184
172
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatNumber, htmlToBasicMarkdown, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface MastodonAccount {
|
|
5
5
|
id: string;
|
|
@@ -141,9 +141,9 @@ function formatStatus(status: MastodonStatus, isReblog = false): string {
|
|
|
141
141
|
|
|
142
142
|
// Stats
|
|
143
143
|
md += `---\n`;
|
|
144
|
-
md += `💬 ${
|
|
145
|
-
md += `🔁 ${
|
|
146
|
-
md += `⭐ ${
|
|
144
|
+
md += `💬 ${formatNumber(status.replies_count)} replies · `;
|
|
145
|
+
md += `🔁 ${formatNumber(status.reblogs_count)} boosts · `;
|
|
146
|
+
md += `⭐ ${formatNumber(status.favourites_count)} favorites\n`;
|
|
147
147
|
|
|
148
148
|
return md;
|
|
149
149
|
}
|
|
@@ -167,9 +167,9 @@ function formatAccount(account: MastodonAccount): string {
|
|
|
167
167
|
}
|
|
168
168
|
|
|
169
169
|
// Stats
|
|
170
|
-
md += `**Followers:** ${
|
|
171
|
-
md += `**Following:** ${
|
|
172
|
-
md += `**Posts:** ${
|
|
170
|
+
md += `**Followers:** ${formatNumber(account.followers_count)} · `;
|
|
171
|
+
md += `**Following:** ${formatNumber(account.following_count)} · `;
|
|
172
|
+
md += `**Posts:** ${formatNumber(account.statuses_count)}\n\n`;
|
|
173
173
|
|
|
174
174
|
md += `**Joined:** ${formatDate(account.created_at)}\n`;
|
|
175
175
|
md += `**Profile:** ${account.url}\n`;
|
|
@@ -224,26 +224,18 @@ export const handleMastodon: SpecialHandler = async (
|
|
|
224
224
|
|
|
225
225
|
if (!result.ok) return null;
|
|
226
226
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
status = JSON.parse(result.content);
|
|
230
|
-
} catch {
|
|
231
|
-
return null;
|
|
232
|
-
}
|
|
227
|
+
const status = tryParseJson<MastodonStatus>(result.content);
|
|
228
|
+
if (!status) return null;
|
|
233
229
|
|
|
234
230
|
const md = formatStatus(status);
|
|
235
|
-
const output = finalizeOutput(md);
|
|
236
231
|
|
|
237
|
-
return {
|
|
232
|
+
return buildResult(md, {
|
|
238
233
|
url,
|
|
239
234
|
finalUrl: status.url || url,
|
|
240
|
-
contentType: "text/markdown",
|
|
241
235
|
method: "mastodon",
|
|
242
|
-
content: output.content,
|
|
243
236
|
fetchedAt,
|
|
244
|
-
truncated: output.truncated,
|
|
245
237
|
notes: [`Fetched via Mastodon API (${instance})`],
|
|
246
|
-
};
|
|
238
|
+
});
|
|
247
239
|
}
|
|
248
240
|
|
|
249
241
|
if (profileMatch) {
|
|
@@ -259,12 +251,8 @@ export const handleMastodon: SpecialHandler = async (
|
|
|
259
251
|
|
|
260
252
|
if (!result.ok) return null;
|
|
261
253
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
account = JSON.parse(result.content);
|
|
265
|
-
} catch {
|
|
266
|
-
return null;
|
|
267
|
-
}
|
|
254
|
+
const account = tryParseJson<MastodonAccount>(result.content);
|
|
255
|
+
if (!account) return null;
|
|
268
256
|
|
|
269
257
|
// Fetch recent statuses
|
|
270
258
|
const statusesUrl = `https://${instance}/api/v1/accounts/${account.id}/statuses?limit=5&exclude_replies=true`;
|
|
@@ -277,32 +265,25 @@ export const handleMastodon: SpecialHandler = async (
|
|
|
277
265
|
let md = formatAccount(account);
|
|
278
266
|
|
|
279
267
|
if (statusesResult.ok) {
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
md += `💬 ${status.replies_count} · 🔁 ${status.reblogs_count} · ⭐ ${status.favourites_count}\n\n`;
|
|
289
|
-
}
|
|
268
|
+
const statuses = tryParseJson<MastodonStatus[]>(statusesResult.content);
|
|
269
|
+
if (statuses && statuses.length > 0) {
|
|
270
|
+
md += "\n---\n\n## Recent Posts\n\n";
|
|
271
|
+
for (const status of statuses.slice(0, 5)) {
|
|
272
|
+
md += `### ${formatDate(status.created_at)}\n\n`;
|
|
273
|
+
const content = htmlToBasicMarkdown(status.content);
|
|
274
|
+
md += `${content}\n\n`;
|
|
275
|
+
md += `\uD83D\uDCAC ${status.replies_count} \u00B7 \uD83D\uDD01 ${status.reblogs_count} \u00B7 \u2B50 ${status.favourites_count}\n\n`;
|
|
290
276
|
}
|
|
291
|
-
}
|
|
277
|
+
}
|
|
292
278
|
}
|
|
293
279
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
return {
|
|
280
|
+
return buildResult(md, {
|
|
297
281
|
url,
|
|
298
282
|
finalUrl: account.url || url,
|
|
299
|
-
contentType: "text/markdown",
|
|
300
283
|
method: "mastodon",
|
|
301
|
-
content: output.content,
|
|
302
284
|
fetchedAt,
|
|
303
|
-
truncated: output.truncated,
|
|
304
285
|
notes: [`Fetched via Mastodon API (${instance})`],
|
|
305
|
-
};
|
|
286
|
+
});
|
|
306
287
|
}
|
|
307
288
|
} catch {}
|
|
308
289
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface MavenDoc {
|
|
5
5
|
id: string;
|
|
@@ -74,12 +74,8 @@ export const handleMaven: SpecialHandler = async (
|
|
|
74
74
|
|
|
75
75
|
if (!result.ok) return null;
|
|
76
76
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
data = JSON.parse(result.content);
|
|
80
|
-
} catch {
|
|
81
|
-
return null;
|
|
82
|
-
}
|
|
77
|
+
const data = tryParseJson<MavenResponse>(result.content);
|
|
78
|
+
if (!data) return null;
|
|
83
79
|
|
|
84
80
|
if (data.response.numFound === 0) return null;
|
|
85
81
|
|
|
@@ -96,10 +92,9 @@ export const handleMaven: SpecialHandler = async (
|
|
|
96
92
|
md += "\n";
|
|
97
93
|
|
|
98
94
|
if (doc.p) md += `**Packaging:** ${doc.p}\n`;
|
|
99
|
-
if (doc.versionCount) md += `**Versions:** ${
|
|
95
|
+
if (doc.versionCount) md += `**Versions:** ${formatNumber(doc.versionCount)}\n`;
|
|
100
96
|
if (doc.timestamp) {
|
|
101
|
-
|
|
102
|
-
md += `**Last Updated:** ${date.toISOString().split("T")[0]}\n`;
|
|
97
|
+
md += `**Last Updated:** ${formatIsoDate(doc.timestamp)}\n`;
|
|
103
98
|
}
|
|
104
99
|
|
|
105
100
|
// Add dependency snippets
|
|
@@ -135,17 +130,7 @@ export const handleMaven: SpecialHandler = async (
|
|
|
135
130
|
md += `- [Maven Central](https://search.maven.org/artifact/${doc.g}/${doc.a}/${displayVersion}/jar)\n`;
|
|
136
131
|
md += `- [MVN Repository](https://mvnrepository.com/artifact/${doc.g}/${doc.a}/${displayVersion})\n`;
|
|
137
132
|
|
|
138
|
-
|
|
139
|
-
return {
|
|
140
|
-
url,
|
|
141
|
-
finalUrl: url,
|
|
142
|
-
contentType: "text/markdown",
|
|
143
|
-
method: "maven",
|
|
144
|
-
content: output.content,
|
|
145
|
-
fetchedAt,
|
|
146
|
-
truncated: output.truncated,
|
|
147
|
-
notes: ["Fetched via Maven Central API"],
|
|
148
|
-
};
|
|
133
|
+
return buildResult(md, { url, method: "maven", fetchedAt, notes: ["Fetched via Maven Central API"] });
|
|
149
134
|
} catch {}
|
|
150
135
|
|
|
151
136
|
return null;
|
package/src/web/scrapers/mdn.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, htmlToBasicMarkdown, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface MDNSection {
|
|
5
5
|
type: string;
|
|
@@ -129,14 +129,14 @@ export const handleMDN: SpecialHandler = async (url: string, timeout: number, si
|
|
|
129
129
|
return null;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
-
const data
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
if (!doc || !doc.title) {
|
|
132
|
+
const data = tryParseJson<MDNDoc>(result.content);
|
|
133
|
+
if (!data?.doc?.title) {
|
|
136
134
|
notes.push("Invalid MDN JSON structure");
|
|
137
135
|
return null;
|
|
138
136
|
}
|
|
139
137
|
|
|
138
|
+
const { doc } = data;
|
|
139
|
+
|
|
140
140
|
// Build markdown content
|
|
141
141
|
const parts: string[] = [];
|
|
142
142
|
|
|
@@ -153,18 +153,14 @@ export const handleMDN: SpecialHandler = async (url: string, timeout: number, si
|
|
|
153
153
|
}
|
|
154
154
|
|
|
155
155
|
const rawContent = parts.join("\n\n");
|
|
156
|
-
const { content, truncated } = finalizeOutput(rawContent);
|
|
157
156
|
|
|
158
|
-
return {
|
|
157
|
+
return buildResult(rawContent, {
|
|
159
158
|
url,
|
|
160
159
|
finalUrl: doc.mdn_url || result.finalUrl,
|
|
161
|
-
contentType: "text/markdown",
|
|
162
160
|
method: "mdn",
|
|
163
|
-
content,
|
|
164
161
|
fetchedAt: new Date().toISOString(),
|
|
165
|
-
truncated,
|
|
166
162
|
notes,
|
|
167
|
-
};
|
|
163
|
+
});
|
|
168
164
|
} catch (err) {
|
|
169
165
|
notes.push(`MDN handler error: ${err instanceof Error ? err.message : String(err)}`);
|
|
170
166
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface ModuleResponse {
|
|
5
5
|
name: string;
|
|
@@ -89,12 +89,8 @@ async function fetchModule(
|
|
|
89
89
|
|
|
90
90
|
if (!result.ok) return null;
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
module = JSON.parse(result.content);
|
|
95
|
-
} catch {
|
|
96
|
-
return null;
|
|
97
|
-
}
|
|
92
|
+
const module = tryParseJson<ModuleResponse>(result.content);
|
|
93
|
+
if (!module) return null;
|
|
98
94
|
|
|
99
95
|
// Fetch additional release info for dependencies and metadata
|
|
100
96
|
const releaseUrl = `https://fastapi.metacpan.org/v1/release/${module.distribution}`;
|
|
@@ -102,24 +98,11 @@ async function fetchModule(
|
|
|
102
98
|
|
|
103
99
|
let release: ReleaseResponse | null = null;
|
|
104
100
|
if (releaseResult.ok) {
|
|
105
|
-
|
|
106
|
-
release = JSON.parse(releaseResult.content);
|
|
107
|
-
} catch {}
|
|
101
|
+
release = tryParseJson<ReleaseResponse>(releaseResult.content);
|
|
108
102
|
}
|
|
109
103
|
|
|
110
104
|
const md = formatModuleMarkdown(module, release);
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
return {
|
|
114
|
-
url,
|
|
115
|
-
finalUrl: url,
|
|
116
|
-
contentType: "text/markdown",
|
|
117
|
-
method: "metacpan",
|
|
118
|
-
content: output.content,
|
|
119
|
-
fetchedAt,
|
|
120
|
-
truncated: output.truncated,
|
|
121
|
-
notes: ["Fetched via MetaCPAN API"],
|
|
122
|
-
};
|
|
105
|
+
return buildResult(md, { url, method: "metacpan", fetchedAt, notes: ["Fetched via MetaCPAN API"] });
|
|
123
106
|
}
|
|
124
107
|
|
|
125
108
|
async function fetchRelease(
|
|
@@ -134,26 +117,11 @@ async function fetchRelease(
|
|
|
134
117
|
|
|
135
118
|
if (!result.ok) return null;
|
|
136
119
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
release = JSON.parse(result.content);
|
|
140
|
-
} catch {
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
120
|
+
const release = tryParseJson<ReleaseResponse>(result.content);
|
|
121
|
+
if (!release) return null;
|
|
143
122
|
|
|
144
123
|
const md = formatReleaseMarkdown(release);
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
return {
|
|
148
|
-
url,
|
|
149
|
-
finalUrl: url,
|
|
150
|
-
contentType: "text/markdown",
|
|
151
|
-
method: "metacpan",
|
|
152
|
-
content: output.content,
|
|
153
|
-
fetchedAt,
|
|
154
|
-
truncated: output.truncated,
|
|
155
|
-
notes: ["Fetched via MetaCPAN API"],
|
|
156
|
-
};
|
|
124
|
+
return buildResult(md, { url, method: "metacpan", fetchedAt, notes: ["Fetched via MetaCPAN API"] });
|
|
157
125
|
}
|
|
158
126
|
|
|
159
127
|
function formatModuleMarkdown(module: ModuleResponse, release: ReleaseResponse | null): string {
|
|
@@ -215,7 +183,7 @@ function formatReleaseMarkdown(release: ReleaseResponse): string {
|
|
|
215
183
|
}
|
|
216
184
|
|
|
217
185
|
if (release.stat?.mtime) {
|
|
218
|
-
const date =
|
|
186
|
+
const date = formatIsoDate(release.stat.mtime * 1000);
|
|
219
187
|
md += `**Released:** ${date}\n`;
|
|
220
188
|
}
|
|
221
189
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* MusicBrainz URL handler for artists, releases, and recordings
|
|
3
3
|
*/
|
|
4
4
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
5
|
-
import {
|
|
5
|
+
import { buildResult, formatMediaDuration, loadPage, tryParseJson } from "./types";
|
|
6
6
|
|
|
7
7
|
type MusicBrainzEntity = "artist" | "release" | "recording";
|
|
8
8
|
|
|
@@ -92,11 +92,7 @@ async function fetchJson<T>(apiUrl: string, timeout: number, signal?: AbortSigna
|
|
|
92
92
|
|
|
93
93
|
if (!result.ok) return null;
|
|
94
94
|
|
|
95
|
-
|
|
96
|
-
return JSON.parse(result.content) as T;
|
|
97
|
-
} catch {
|
|
98
|
-
return null;
|
|
99
|
-
}
|
|
95
|
+
return tryParseJson<T>(result.content);
|
|
100
96
|
}
|
|
101
97
|
|
|
102
98
|
function formatLifeSpan(life: MusicBrainzLifeSpan | undefined): string | null {
|
|
@@ -115,17 +111,7 @@ function formatLifeSpan(life: MusicBrainzLifeSpan | undefined): string | null {
|
|
|
115
111
|
|
|
116
112
|
function formatDurationMs(lengthMs: number | undefined): string | null {
|
|
117
113
|
if (!lengthMs || lengthMs <= 0) return null;
|
|
118
|
-
|
|
119
|
-
const totalSeconds = Math.round(lengthMs / 1000);
|
|
120
|
-
const hours = Math.floor(totalSeconds / 3600);
|
|
121
|
-
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
|
122
|
-
const seconds = totalSeconds % 60;
|
|
123
|
-
|
|
124
|
-
if (hours > 0) {
|
|
125
|
-
return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
|
|
114
|
+
return formatMediaDuration(Math.round(lengthMs / 1000));
|
|
129
115
|
}
|
|
130
116
|
|
|
131
117
|
function formatArtistCredits(credits: MusicBrainzArtistCredit[] | undefined): string | null {
|
|
@@ -255,17 +241,7 @@ export const handleMusicBrainz: SpecialHandler = async (
|
|
|
255
241
|
md = buildRecordingMarkdown(recording);
|
|
256
242
|
}
|
|
257
243
|
|
|
258
|
-
|
|
259
|
-
return {
|
|
260
|
-
url,
|
|
261
|
-
finalUrl: url,
|
|
262
|
-
contentType: "text/markdown",
|
|
263
|
-
method: "musicbrainz-api",
|
|
264
|
-
content: output.content,
|
|
265
|
-
fetchedAt,
|
|
266
|
-
truncated: output.truncated,
|
|
267
|
-
notes: ["Fetched via MusicBrainz API"],
|
|
268
|
-
};
|
|
244
|
+
return buildResult(md, { url, method: "musicbrainz-api", fetchedAt, notes: ["Fetched via MusicBrainz API"] });
|
|
269
245
|
} catch {}
|
|
270
246
|
|
|
271
247
|
return null;
|
package/src/web/scrapers/npm.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handle npm URLs via registry API
|
|
@@ -41,13 +41,11 @@ export const handleNpm: SpecialHandler = async (
|
|
|
41
41
|
// Parse download stats
|
|
42
42
|
let weeklyDownloads: number | null = null;
|
|
43
43
|
if (downloadsResult.ok) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
weeklyDownloads = dlData.downloads ?? null;
|
|
47
|
-
} catch {}
|
|
44
|
+
const dlData = tryParseJson<{ downloads?: number }>(downloadsResult.content);
|
|
45
|
+
if (dlData) weeklyDownloads = dlData.downloads ?? null;
|
|
48
46
|
}
|
|
49
47
|
|
|
50
|
-
|
|
48
|
+
const pkg = tryParseJson<{
|
|
51
49
|
name: string;
|
|
52
50
|
version: string;
|
|
53
51
|
description?: string;
|
|
@@ -58,13 +56,8 @@ export const handleNpm: SpecialHandler = async (
|
|
|
58
56
|
maintainers?: Array<{ name: string }>;
|
|
59
57
|
dependencies?: Record<string, string>;
|
|
60
58
|
readme?: string;
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
try {
|
|
64
|
-
pkg = JSON.parse(result.content);
|
|
65
|
-
} catch {
|
|
66
|
-
return null; // JSON parse failed (truncated response)
|
|
67
|
-
}
|
|
59
|
+
}>(result.content);
|
|
60
|
+
if (!pkg) return null;
|
|
68
61
|
|
|
69
62
|
let md = `# ${pkg.name}\n\n`;
|
|
70
63
|
if (pkg.description) md += `${pkg.description}\n\n`;
|
|
@@ -76,7 +69,7 @@ export const handleNpm: SpecialHandler = async (
|
|
|
76
69
|
}
|
|
77
70
|
md += "\n";
|
|
78
71
|
if (weeklyDownloads !== null) {
|
|
79
|
-
md += `**Weekly Downloads:** ${
|
|
72
|
+
md += `**Weekly Downloads:** ${formatNumber(weeklyDownloads)}\n`;
|
|
80
73
|
}
|
|
81
74
|
md += "\n";
|
|
82
75
|
|
|
@@ -97,17 +90,7 @@ export const handleNpm: SpecialHandler = async (
|
|
|
97
90
|
md += `\n---\n\n## README\n\n${pkg.readme}\n`;
|
|
98
91
|
}
|
|
99
92
|
|
|
100
|
-
|
|
101
|
-
return {
|
|
102
|
-
url,
|
|
103
|
-
finalUrl: url,
|
|
104
|
-
contentType: "text/markdown",
|
|
105
|
-
method: "npm",
|
|
106
|
-
content: output.content,
|
|
107
|
-
fetchedAt,
|
|
108
|
-
truncated: output.truncated,
|
|
109
|
-
notes: ["Fetched via npm registry"],
|
|
110
|
-
};
|
|
93
|
+
return buildResult(md, { url, method: "npm", fetchedAt, notes: ["Fetched via npm registry"] });
|
|
111
94
|
} catch {}
|
|
112
95
|
|
|
113
96
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface NuGetCatalogEntry {
|
|
5
5
|
id: string;
|
|
@@ -60,12 +60,8 @@ export const handleNuGet: SpecialHandler = async (
|
|
|
60
60
|
|
|
61
61
|
if (!result.ok) return null;
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
index = JSON.parse(result.content);
|
|
66
|
-
} catch {
|
|
67
|
-
return null;
|
|
68
|
-
}
|
|
63
|
+
const index = tryParseJson<NuGetRegistrationIndex>(result.content);
|
|
64
|
+
if (!index) return null;
|
|
69
65
|
|
|
70
66
|
if (!index.items?.length) return null;
|
|
71
67
|
|
|
@@ -76,11 +72,9 @@ export const handleNuGet: SpecialHandler = async (
|
|
|
76
72
|
if (!latestPage.items && latestPage["@id"]) {
|
|
77
73
|
const pageResult = await loadPage(latestPage["@id"], { timeout, signal });
|
|
78
74
|
if (!pageResult.ok) return null;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
return null;
|
|
83
|
-
}
|
|
75
|
+
const fetched = tryParseJson<NuGetRegistrationPage>(pageResult.content);
|
|
76
|
+
if (!fetched) return null;
|
|
77
|
+
latestPage = fetched;
|
|
84
78
|
}
|
|
85
79
|
|
|
86
80
|
if (!latestPage.items?.length) return null;
|
|
@@ -97,10 +91,8 @@ export const handleNuGet: SpecialHandler = async (
|
|
|
97
91
|
if (!pageItems && page["@id"]) {
|
|
98
92
|
const pageResult = await loadPage(page["@id"], { timeout: Math.min(timeout, 5), signal });
|
|
99
93
|
if (pageResult.ok) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
pageItems = fetchedPage.items;
|
|
103
|
-
} catch {}
|
|
94
|
+
const fetchedPage = tryParseJson<NuGetRegistrationPage>(pageResult.content);
|
|
95
|
+
if (fetchedPage) pageItems = fetchedPage.items;
|
|
104
96
|
}
|
|
105
97
|
}
|
|
106
98
|
|
|
@@ -128,12 +120,8 @@ export const handleNuGet: SpecialHandler = async (
|
|
|
128
120
|
const searchResult = await loadPage(searchUrl, { timeout: Math.min(timeout, 5), signal });
|
|
129
121
|
|
|
130
122
|
if (searchResult.ok) {
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
data?: Array<{ totalDownloads?: number }>;
|
|
134
|
-
};
|
|
135
|
-
totalDownloads = searchData.data?.[0]?.totalDownloads ?? null;
|
|
136
|
-
} catch {}
|
|
123
|
+
const searchData = tryParseJson<{ data?: Array<{ totalDownloads?: number }> }>(searchResult.content);
|
|
124
|
+
if (searchData) totalDownloads = searchData.data?.[0]?.totalDownloads ?? null;
|
|
137
125
|
}
|
|
138
126
|
|
|
139
127
|
// Format markdown output
|
|
@@ -149,15 +137,14 @@ export const handleNuGet: SpecialHandler = async (
|
|
|
149
137
|
md += "\n";
|
|
150
138
|
|
|
151
139
|
if (totalDownloads !== null) {
|
|
152
|
-
md += `**Total Downloads:** ${
|
|
140
|
+
md += `**Total Downloads:** ${formatNumber(totalDownloads)}\n`;
|
|
153
141
|
}
|
|
154
142
|
|
|
155
143
|
if (targetEntry.authors) md += `**Authors:** ${targetEntry.authors}\n`;
|
|
156
144
|
if (targetEntry.projectUrl) md += `**Project URL:** ${targetEntry.projectUrl}\n`;
|
|
157
145
|
if (targetEntry.tags?.length) md += `**Tags:** ${targetEntry.tags.join(", ")}\n`;
|
|
158
146
|
if (targetEntry.published) {
|
|
159
|
-
|
|
160
|
-
md += `**Published:** ${pubDate}\n`;
|
|
147
|
+
md += `**Published:** ${formatIsoDate(targetEntry.published)}\n`;
|
|
161
148
|
}
|
|
162
149
|
|
|
163
150
|
// Show dependencies by target framework
|
|
@@ -183,22 +170,12 @@ export const handleNuGet: SpecialHandler = async (
|
|
|
183
170
|
const recentVersions = latestPage.items.slice(-5).reverse();
|
|
184
171
|
for (const item of recentVersions) {
|
|
185
172
|
const entry = item.catalogEntry;
|
|
186
|
-
const pubDate = entry.published
|
|
173
|
+
const pubDate = formatIsoDate(entry.published) || "unknown";
|
|
187
174
|
md += `- **${entry.version}** (${pubDate})\n`;
|
|
188
175
|
}
|
|
189
176
|
}
|
|
190
177
|
|
|
191
|
-
|
|
192
|
-
return {
|
|
193
|
-
url,
|
|
194
|
-
finalUrl: url,
|
|
195
|
-
contentType: "text/markdown",
|
|
196
|
-
method: "nuget",
|
|
197
|
-
content: output.content,
|
|
198
|
-
fetchedAt,
|
|
199
|
-
truncated: output.truncated,
|
|
200
|
-
notes: ["Fetched via NuGet API"],
|
|
201
|
-
};
|
|
178
|
+
return buildResult(md, { url, method: "nuget", fetchedAt, notes: ["Fetched via NuGet API"] });
|
|
202
179
|
} catch {}
|
|
203
180
|
|
|
204
181
|
return null;
|