@oh-my-pi/pi-coding-agent 12.18.3 → 12.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/package.json +35 -27
- package/src/async/index.ts +1 -0
- package/src/async/job-manager.ts +341 -0
- package/src/cli/file-processor.ts +3 -3
- package/src/cli/list-models.ts +3 -17
- package/src/cli/stats-cli.ts +3 -22
- package/src/cli/web-search-cli.ts +8 -16
- package/src/commit/agentic/agent.ts +6 -9
- package/src/commit/agentic/index.ts +44 -50
- package/src/commit/agentic/state.ts +0 -9
- package/src/commit/agentic/tools/propose-commit.ts +1 -30
- package/src/commit/agentic/tools/schemas.ts +31 -0
- package/src/commit/agentic/tools/split-commit.ts +1 -30
- package/src/commit/agentic/validation.ts +1 -18
- package/src/commit/analysis/conventional.ts +3 -50
- package/src/commit/analysis/summary.ts +2 -13
- package/src/commit/changelog/detect.ts +4 -1
- package/src/commit/changelog/generate.ts +2 -25
- package/src/commit/changelog/index.ts +1 -2
- package/src/commit/cli.ts +4 -12
- package/src/commit/map-reduce/reduce-phase.ts +2 -43
- package/src/commit/pipeline.ts +7 -15
- package/src/commit/utils.ts +44 -0
- package/src/config/prompt-templates.ts +1 -81
- package/src/config/settings-schema.ts +20 -1
- package/src/config.ts +2 -3
- package/src/debug/index.ts +1 -6
- package/src/debug/system-info.ts +2 -6
- package/src/discovery/builtin.ts +5 -9
- package/src/discovery/helpers.ts +0 -26
- package/src/discovery/ssh.ts +1 -8
- package/src/exa/company.ts +8 -39
- package/src/exa/factory.ts +64 -0
- package/src/exa/index.ts +0 -16
- package/src/exa/linkedin.ts +8 -39
- package/src/exa/mcp-client.ts +0 -64
- package/src/exa/researcher.ts +17 -59
- package/src/exa/search.ts +30 -154
- package/src/extensibility/custom-tools/loader.ts +3 -41
- package/src/extensibility/extensions/loader.ts +2 -9
- package/src/extensibility/hooks/loader.ts +3 -20
- package/src/extensibility/hooks/runner.ts +3 -19
- package/src/extensibility/plugins/installer.ts +2 -1
- package/src/extensibility/plugins/loader.ts +29 -117
- package/src/extensibility/skills.ts +2 -89
- package/src/extensibility/slash-commands.ts +1 -63
- package/src/extensibility/utils.ts +38 -0
- package/src/index.ts +9 -25
- package/src/internal-urls/index.ts +1 -0
- package/src/internal-urls/jobs-protocol.ts +118 -0
- package/src/ipy/kernel.ts +2 -0
- package/src/lsp/config.ts +1 -5
- package/src/lsp/lspmux.ts +0 -17
- package/src/lsp/utils.ts +2 -24
- package/src/main.ts +16 -24
- package/src/mcp/client.ts +1 -46
- package/src/mcp/render.ts +8 -1
- package/src/mcp/tool-cache.ts +1 -5
- package/src/mcp/transports/http.ts +2 -7
- package/src/mcp/transports/stdio.ts +2 -7
- package/src/modes/components/bash-execution.ts +2 -16
- package/src/modes/components/extensions/inspector-panel.ts +8 -18
- package/src/modes/components/footer.ts +10 -50
- package/src/modes/components/model-selector.ts +2 -21
- package/src/modes/components/python-execution.ts +2 -16
- package/src/modes/components/settings-selector.ts +1 -10
- package/src/modes/components/status-line/segments.ts +8 -25
- package/src/modes/components/status-line.ts +14 -31
- package/src/modes/components/tool-execution.ts +8 -2
- package/src/modes/controllers/command-controller.ts +71 -30
- package/src/modes/controllers/event-controller.ts +34 -4
- package/src/modes/controllers/mcp-command-controller.ts +3 -34
- package/src/modes/controllers/selector-controller.ts +2 -2
- package/src/modes/controllers/ssh-command-controller.ts +3 -34
- package/src/modes/interactive-mode.ts +6 -2
- package/src/modes/rpc/rpc-client.ts +1 -5
- package/src/modes/shared.ts +73 -0
- package/src/modes/types.ts +1 -0
- package/src/modes/utils/ui-helpers.ts +26 -2
- package/src/patch/hashline.ts +6 -286
- package/src/patch/index.ts +6 -57
- package/src/patch/normalize.ts +22 -65
- package/src/patch/shared.ts +16 -16
- package/src/prompts/system/custom-system-prompt.md +0 -10
- package/src/prompts/system/system-prompt.md +69 -89
- package/src/prompts/tools/async-result.md +5 -0
- package/src/prompts/tools/bash.md +5 -0
- package/src/prompts/tools/cancel-job.md +7 -0
- package/src/prompts/tools/hashline.md +0 -16
- package/src/prompts/tools/poll-jobs.md +7 -0
- package/src/prompts/tools/task.md +4 -0
- package/src/sdk.ts +70 -6
- package/src/session/agent-session.ts +43 -6
- package/src/session/agent-storage.ts +69 -278
- package/src/session/auth-storage.ts +14 -1430
- package/src/session/session-manager.ts +69 -5
- package/src/session/session-storage.ts +1 -5
- package/src/session/streaming-output.ts +637 -76
- package/src/slash-commands/builtin-registry.ts +8 -0
- package/src/ssh/connection-manager.ts +4 -12
- package/src/ssh/sshfs-mount.ts +3 -7
- package/src/ssh/utils.ts +8 -0
- package/src/system-prompt.ts +24 -90
- package/src/task/executor.ts +11 -1
- package/src/task/index.ts +258 -13
- package/src/task/parallel.ts +32 -0
- package/src/task/render.ts +15 -7
- package/src/task/types.ts +5 -0
- package/src/tools/ask.ts +4 -7
- package/src/tools/bash-interactive.ts +4 -5
- package/src/tools/bash.ts +125 -41
- package/src/tools/cancel-job.ts +93 -0
- package/src/tools/fetch.ts +7 -27
- package/src/tools/find.ts +3 -3
- package/src/tools/gemini-image.ts +15 -14
- package/src/tools/grep.ts +3 -3
- package/src/tools/index.ts +13 -29
- package/src/tools/json-tree.ts +12 -1
- package/src/tools/jtd-to-json-schema.ts +10 -74
- package/src/tools/jtd-to-typescript.ts +10 -72
- package/src/tools/jtd-utils.ts +102 -0
- package/src/tools/notebook.ts +4 -9
- package/src/tools/output-meta.ts +52 -26
- package/src/tools/path-utils.ts +13 -7
- package/src/tools/poll-jobs.ts +178 -0
- package/src/tools/python.ts +32 -35
- package/src/tools/read.ts +61 -82
- package/src/tools/render-utils.ts +8 -159
- package/src/tools/ssh.ts +7 -20
- package/src/tools/submit-result.ts +1 -1
- package/src/tools/tool-errors.ts +0 -30
- package/src/tools/tool-result.ts +1 -2
- package/src/tools/write.ts +8 -10
- package/src/tui/code-cell.ts +8 -3
- package/src/tui/status-line.ts +4 -4
- package/src/tui/types.ts +0 -1
- package/src/tui/utils.ts +1 -14
- package/src/utils/command-args.ts +76 -0
- package/src/utils/file-mentions.ts +15 -19
- package/src/utils/frontmatter.ts +5 -10
- package/src/utils/shell-snapshot.ts +0 -11
- package/src/utils/title-generator.ts +0 -12
- package/src/web/scrapers/artifacthub.ts +7 -16
- package/src/web/scrapers/arxiv.ts +3 -8
- package/src/web/scrapers/aur.ts +8 -22
- package/src/web/scrapers/biorxiv.ts +5 -14
- package/src/web/scrapers/bluesky.ts +13 -36
- package/src/web/scrapers/brew.ts +5 -10
- package/src/web/scrapers/cheatsh.ts +2 -12
- package/src/web/scrapers/chocolatey.ts +63 -26
- package/src/web/scrapers/choosealicense.ts +3 -18
- package/src/web/scrapers/cisa-kev.ts +4 -18
- package/src/web/scrapers/clojars.ts +6 -33
- package/src/web/scrapers/coingecko.ts +25 -33
- package/src/web/scrapers/crates-io.ts +7 -26
- package/src/web/scrapers/crossref.ts +4 -18
- package/src/web/scrapers/devto.ts +11 -41
- package/src/web/scrapers/discogs.ts +7 -10
- package/src/web/scrapers/discourse.ts +6 -31
- package/src/web/scrapers/dockerhub.ts +12 -35
- package/src/web/scrapers/fdroid.ts +8 -33
- package/src/web/scrapers/firefox-addons.ts +10 -34
- package/src/web/scrapers/flathub.ts +7 -24
- package/src/web/scrapers/github-gist.ts +2 -12
- package/src/web/scrapers/github.ts +9 -47
- package/src/web/scrapers/gitlab.ts +130 -185
- package/src/web/scrapers/go-pkg.ts +12 -22
- package/src/web/scrapers/hackage.ts +88 -43
- package/src/web/scrapers/hackernews.ts +25 -45
- package/src/web/scrapers/hex.ts +19 -36
- package/src/web/scrapers/huggingface.ts +26 -91
- package/src/web/scrapers/iacr.ts +3 -8
- package/src/web/scrapers/jetbrains-marketplace.ts +9 -20
- package/src/web/scrapers/lemmy.ts +5 -23
- package/src/web/scrapers/lobsters.ts +16 -28
- package/src/web/scrapers/mastodon.ts +24 -43
- package/src/web/scrapers/maven.ts +6 -21
- package/src/web/scrapers/mdn.ts +7 -11
- package/src/web/scrapers/metacpan.ts +9 -41
- package/src/web/scrapers/musicbrainz.ts +4 -28
- package/src/web/scrapers/npm.ts +8 -25
- package/src/web/scrapers/nuget.ts +14 -37
- package/src/web/scrapers/nvd.ts +6 -28
- package/src/web/scrapers/ollama.ts +7 -34
- package/src/web/scrapers/open-vsx.ts +5 -19
- package/src/web/scrapers/opencorporates.ts +30 -14
- package/src/web/scrapers/openlibrary.ts +49 -33
- package/src/web/scrapers/orcid.ts +4 -18
- package/src/web/scrapers/osv.ts +7 -24
- package/src/web/scrapers/packagist.ts +9 -24
- package/src/web/scrapers/pub-dev.ts +7 -50
- package/src/web/scrapers/pubmed.ts +54 -21
- package/src/web/scrapers/pypi.ts +8 -26
- package/src/web/scrapers/rawg.ts +11 -19
- package/src/web/scrapers/readthedocs.ts +4 -9
- package/src/web/scrapers/reddit.ts +5 -15
- package/src/web/scrapers/repology.ts +8 -20
- package/src/web/scrapers/rfc.ts +5 -14
- package/src/web/scrapers/rubygems.ts +6 -21
- package/src/web/scrapers/searchcode.ts +8 -36
- package/src/web/scrapers/sec-edgar.ts +4 -18
- package/src/web/scrapers/semantic-scholar.ts +15 -35
- package/src/web/scrapers/snapcraft.ts +5 -19
- package/src/web/scrapers/sourcegraph.ts +5 -43
- package/src/web/scrapers/spdx.ts +4 -18
- package/src/web/scrapers/spotify.ts +4 -23
- package/src/web/scrapers/stackoverflow.ts +8 -13
- package/src/web/scrapers/terraform.ts +9 -37
- package/src/web/scrapers/tldr.ts +3 -7
- package/src/web/scrapers/twitter.ts +3 -7
- package/src/web/scrapers/types.ts +105 -27
- package/src/web/scrapers/utils.ts +97 -103
- package/src/web/scrapers/vimeo.ts +7 -27
- package/src/web/scrapers/vscode-marketplace.ts +8 -17
- package/src/web/scrapers/w3c.ts +6 -14
- package/src/web/scrapers/wikidata.ts +5 -19
- package/src/web/scrapers/wikipedia.ts +2 -12
- package/src/web/scrapers/youtube.ts +5 -34
- package/src/web/search/index.ts +0 -9
- package/src/web/search/providers/anthropic.ts +3 -2
- package/src/web/search/providers/brave.ts +3 -18
- package/src/web/search/providers/exa.ts +1 -12
- package/src/web/search/providers/kimi.ts +5 -44
- package/src/web/search/providers/perplexity.ts +1 -12
- package/src/web/search/providers/synthetic.ts +3 -26
- package/src/web/search/providers/utils.ts +36 -0
- package/src/web/search/providers/zai.ts +9 -50
- package/src/web/search/types.ts +0 -28
- package/src/web/search/utils.ts +17 -0
- package/src/tools/output-utils.ts +0 -63
- package/src/tools/truncate.ts +0 -385
- package/src/web/search/auth.ts +0 -178
package/src/utils/frontmatter.ts
CHANGED
|
@@ -1,18 +1,10 @@
|
|
|
1
|
-
import { logger } from "@oh-my-pi/pi-utils";
|
|
1
|
+
import { logger, truncate } from "@oh-my-pi/pi-utils";
|
|
2
2
|
import { YAML } from "bun";
|
|
3
3
|
|
|
4
4
|
function stripHtmlComments(content: string): string {
|
|
5
5
|
return content.replace(/<!--[\s\S]*?-->/g, "");
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
-
function toError(value: unknown): Error {
|
|
9
|
-
return value instanceof Error ? value : new Error(String(`YAML: ${value}`));
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
function truncate(content: string, maxLength: number): string {
|
|
13
|
-
return content.length > maxLength ? `${content.slice(0, maxLength)}…` : content;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
8
|
export class FrontmatterError extends Error {
|
|
17
9
|
constructor(
|
|
18
10
|
error: Error,
|
|
@@ -80,7 +72,10 @@ export function parseFrontmatter(
|
|
|
80
72
|
const loaded = YAML.parse(metadata.replaceAll("\t", " ")) as Record<string, unknown> | null;
|
|
81
73
|
return { frontmatter: Object.assign(frontmatter, loaded), body: body };
|
|
82
74
|
} catch (error) {
|
|
83
|
-
const err = new FrontmatterError(
|
|
75
|
+
const err = new FrontmatterError(
|
|
76
|
+
error instanceof Error ? error : new Error(`YAML: ${error}`),
|
|
77
|
+
loc ?? `Inline '${truncate(content, 64)}'`,
|
|
78
|
+
);
|
|
84
79
|
if (level === "warn" || level === "fatal") {
|
|
85
80
|
logger.warn("Failed to parse YAML frontmatter", { err: err.toString() });
|
|
86
81
|
}
|
|
@@ -180,17 +180,6 @@ export async function getOrCreateSnapshot(
|
|
|
180
180
|
return null;
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
-
/**
|
|
184
|
-
* Get the command prefix to source the snapshot.
|
|
185
|
-
* Returns empty string if no snapshot available.
|
|
186
|
-
*/
|
|
187
|
-
export function getSnapshotSourceCommand(snapshotPath: string | null): string {
|
|
188
|
-
if (!snapshotPath) return "";
|
|
189
|
-
// Escape for shell
|
|
190
|
-
const escaped = snapshotPath.replace(/'/g, "'\\''");
|
|
191
|
-
return `source '${escaped}' 2>/dev/null && `;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
183
|
postmortem.register("shell-snapshot", () => {
|
|
195
184
|
for (const snapshotPath of cachedSnapshotPaths.values()) {
|
|
196
185
|
fs.unlinkSync(snapshotPath);
|
|
@@ -51,18 +51,6 @@ function getTitleModelCandidates(registry: ModelRegistry, savedSmolModel?: strin
|
|
|
51
51
|
return candidates;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
/**
|
|
55
|
-
* Find the best available model for title generation.
|
|
56
|
-
* Uses the configured smol model if set, otherwise auto-discovers using priority chain.
|
|
57
|
-
*
|
|
58
|
-
* @param registry Model registry
|
|
59
|
-
* @param savedSmolModel Optional saved smol model from settings (provider/modelId format)
|
|
60
|
-
*/
|
|
61
|
-
export async function findTitleModel(registry: ModelRegistry, savedSmolModel?: string): Promise<Model<Api> | null> {
|
|
62
|
-
const candidates = getTitleModelCandidates(registry, savedSmolModel);
|
|
63
|
-
return candidates[0] ?? null;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
54
|
/**
|
|
67
55
|
* Generate a title for a session based on the first user message.
|
|
68
56
|
*
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface ArtifactHubMaintainer {
|
|
5
5
|
name: string;
|
|
@@ -79,12 +79,8 @@ export const handleArtifactHub: SpecialHandler = async (
|
|
|
79
79
|
|
|
80
80
|
if (!result.ok) return null;
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
pkg = JSON.parse(result.content);
|
|
85
|
-
} catch {
|
|
86
|
-
return null;
|
|
87
|
-
}
|
|
82
|
+
const pkg = tryParseJson<ArtifactHubPackage>(result.content);
|
|
83
|
+
if (!pkg) return null;
|
|
88
84
|
|
|
89
85
|
const displayName = pkg.display_name || pkg.name;
|
|
90
86
|
const kindLabel = formatKindLabel(kind);
|
|
@@ -103,7 +99,7 @@ export const handleArtifactHub: SpecialHandler = async (
|
|
|
103
99
|
const badges: string[] = [];
|
|
104
100
|
if (pkg.official) badges.push("Official");
|
|
105
101
|
if (pkg.signed) badges.push("Signed");
|
|
106
|
-
if (pkg.stars) badges.push(`${
|
|
102
|
+
if (pkg.stars) badges.push(`${formatNumber(pkg.stars)} stars`);
|
|
107
103
|
if (badges.length > 0) md += `**${badges.join(" · ")}**\n`;
|
|
108
104
|
md += "\n";
|
|
109
105
|
|
|
@@ -153,7 +149,7 @@ export const handleArtifactHub: SpecialHandler = async (
|
|
|
153
149
|
if (pkg.available_versions?.length) {
|
|
154
150
|
md += `\n## Recent Versions\n\n`;
|
|
155
151
|
for (const ver of pkg.available_versions.slice(0, 5)) {
|
|
156
|
-
const date =
|
|
152
|
+
const date = formatIsoDate(ver.ts * 1000);
|
|
157
153
|
md += `- **${ver.version}** (${date})\n`;
|
|
158
154
|
}
|
|
159
155
|
}
|
|
@@ -163,17 +159,12 @@ export const handleArtifactHub: SpecialHandler = async (
|
|
|
163
159
|
md += `\n---\n\n## README\n\n${pkg.readme}\n`;
|
|
164
160
|
}
|
|
165
161
|
|
|
166
|
-
|
|
167
|
-
return {
|
|
162
|
+
return buildResult(md, {
|
|
168
163
|
url,
|
|
169
|
-
finalUrl: url,
|
|
170
|
-
contentType: "text/markdown",
|
|
171
164
|
method: "artifacthub",
|
|
172
|
-
content: output.content,
|
|
173
165
|
fetchedAt,
|
|
174
|
-
truncated: output.truncated,
|
|
175
166
|
notes: [`Fetched via Artifact Hub API (${kindLabel})`],
|
|
176
|
-
};
|
|
167
|
+
});
|
|
177
168
|
} catch {}
|
|
178
169
|
|
|
179
170
|
return null;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parse as parseHtml } from "node-html-parser";
|
|
2
2
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
3
|
-
import {
|
|
3
|
+
import { buildResult, loadPage } from "./types";
|
|
4
4
|
import { convertWithMarkitdown, fetchBinary } from "./utils";
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -71,17 +71,12 @@ export const handleArxiv: SpecialHandler = async (
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
return {
|
|
74
|
+
return buildResult(md, {
|
|
76
75
|
url,
|
|
77
|
-
finalUrl: url,
|
|
78
|
-
contentType: "text/markdown",
|
|
79
76
|
method: "arxiv",
|
|
80
|
-
content: output.content,
|
|
81
77
|
fetchedAt,
|
|
82
|
-
truncated: output.truncated,
|
|
83
78
|
notes: notes.length ? notes : ["Fetched via arXiv API"],
|
|
84
|
-
};
|
|
79
|
+
});
|
|
85
80
|
} catch {}
|
|
86
81
|
|
|
87
82
|
return null;
|
package/src/web/scrapers/aur.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface AurPackage {
|
|
5
5
|
Name: string;
|
|
@@ -57,12 +57,8 @@ export const handleAur: SpecialHandler = async (
|
|
|
57
57
|
|
|
58
58
|
if (!result.ok) return null;
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
data = JSON.parse(result.content);
|
|
63
|
-
} catch {
|
|
64
|
-
return null;
|
|
65
|
-
}
|
|
60
|
+
const data = tryParseJson<AurResponse>(result.content);
|
|
61
|
+
if (!data) return null;
|
|
66
62
|
|
|
67
63
|
if (data.resultcount === 0 || !data.results[0]) return null;
|
|
68
64
|
|
|
@@ -74,7 +70,7 @@ export const handleAur: SpecialHandler = async (
|
|
|
74
70
|
// Package info
|
|
75
71
|
md += `**Version:** ${pkg.Version}`;
|
|
76
72
|
if (pkg.OutOfDate) {
|
|
77
|
-
const outOfDateDate =
|
|
73
|
+
const outOfDateDate = formatIsoDate(pkg.OutOfDate * 1000);
|
|
78
74
|
md += ` (flagged out-of-date: ${outOfDateDate})`;
|
|
79
75
|
}
|
|
80
76
|
md += "\n";
|
|
@@ -85,11 +81,11 @@ export const handleAur: SpecialHandler = async (
|
|
|
85
81
|
md += "**Maintainer:** Orphaned\n";
|
|
86
82
|
}
|
|
87
83
|
|
|
88
|
-
md += `**Votes:** ${
|
|
84
|
+
md += `**Votes:** ${formatNumber(pkg.NumVotes)} · **Popularity:** ${pkg.Popularity.toFixed(2)}\n`;
|
|
89
85
|
|
|
90
86
|
// Timestamps
|
|
91
|
-
const lastModified =
|
|
92
|
-
const firstSubmitted =
|
|
87
|
+
const lastModified = formatIsoDate(pkg.LastModified * 1000);
|
|
88
|
+
const firstSubmitted = formatIsoDate(pkg.FirstSubmitted * 1000);
|
|
93
89
|
md += `**Last Updated:** ${lastModified} · **First Submitted:** ${firstSubmitted}\n`;
|
|
94
90
|
|
|
95
91
|
if (pkg.License?.length) md += `**License:** ${pkg.License.join(", ")}\n`;
|
|
@@ -158,17 +154,7 @@ export const handleAur: SpecialHandler = async (
|
|
|
158
154
|
md += `makepkg -si\n`;
|
|
159
155
|
md += "```\n";
|
|
160
156
|
|
|
161
|
-
|
|
162
|
-
return {
|
|
163
|
-
url,
|
|
164
|
-
finalUrl: url,
|
|
165
|
-
contentType: "text/markdown",
|
|
166
|
-
method: "aur",
|
|
167
|
-
content: output.content,
|
|
168
|
-
fetchedAt,
|
|
169
|
-
truncated: output.truncated,
|
|
170
|
-
notes: ["Fetched via AUR RPC API"],
|
|
171
|
-
};
|
|
157
|
+
return buildResult(md, { url, method: "aur", fetchedAt, notes: ["Fetched via AUR RPC API"] });
|
|
172
158
|
} catch {}
|
|
173
159
|
|
|
174
160
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface BiorxivPaper {
|
|
5
5
|
biorxiv_doi?: string;
|
|
@@ -63,12 +63,8 @@ export const handleBiorxiv: SpecialHandler = async (
|
|
|
63
63
|
|
|
64
64
|
if (!result.ok) return null;
|
|
65
65
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
data = JSON.parse(result.content);
|
|
69
|
-
} catch {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
66
|
+
const data = tryParseJson<BiorxivResponse>(result.content);
|
|
67
|
+
if (!data) return null;
|
|
72
68
|
|
|
73
69
|
if (!data.collection || data.collection.length === 0) return null;
|
|
74
70
|
|
|
@@ -124,17 +120,12 @@ export const handleBiorxiv: SpecialHandler = async (
|
|
|
124
120
|
md += `- [JATS XML](${paper.jatsxml})\n`;
|
|
125
121
|
}
|
|
126
122
|
|
|
127
|
-
|
|
128
|
-
return {
|
|
123
|
+
return buildResult(md, {
|
|
129
124
|
url,
|
|
130
|
-
finalUrl: url,
|
|
131
|
-
contentType: "text/markdown",
|
|
132
125
|
method: server,
|
|
133
|
-
content: output.content,
|
|
134
126
|
fetchedAt: new Date().toISOString(),
|
|
135
|
-
truncated: output.truncated,
|
|
136
127
|
notes: [`Fetched via ${serverName} API`],
|
|
137
|
-
};
|
|
128
|
+
});
|
|
138
129
|
} catch {}
|
|
139
130
|
|
|
140
131
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
const API_BASE = "https://public.api.bsky.app/xrpc";
|
|
5
5
|
|
|
@@ -64,12 +64,9 @@ async function resolveHandle(handle: string, timeout: number, signal?: AbortSign
|
|
|
64
64
|
|
|
65
65
|
if (!result.ok) return null;
|
|
66
66
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
} catch {
|
|
71
|
-
return null;
|
|
72
|
-
}
|
|
67
|
+
const data = tryParseJson<BlueskyProfile>(result.content);
|
|
68
|
+
if (!data) return null;
|
|
69
|
+
return data.did;
|
|
73
70
|
}
|
|
74
71
|
|
|
75
72
|
/**
|
|
@@ -136,10 +133,10 @@ function formatPost(post: BlueskyPost, isQuote = false): string {
|
|
|
136
133
|
// Stats
|
|
137
134
|
if (!isQuote) {
|
|
138
135
|
const stats: string[] = [];
|
|
139
|
-
if (post.likeCount) stats.push(`❤️ ${
|
|
140
|
-
if (post.repostCount) stats.push(`🔁 ${
|
|
141
|
-
if (post.replyCount) stats.push(`💬 ${
|
|
142
|
-
if (post.quoteCount) stats.push(`📝 ${
|
|
136
|
+
if (post.likeCount) stats.push(`❤️ ${formatNumber(post.likeCount)}`);
|
|
137
|
+
if (post.repostCount) stats.push(`🔁 ${formatNumber(post.repostCount)}`);
|
|
138
|
+
if (post.replyCount) stats.push(`💬 ${formatNumber(post.replyCount)}`);
|
|
139
|
+
if (post.quoteCount) stats.push(`📝 ${formatNumber(post.quoteCount)}`);
|
|
143
140
|
if (stats.length) md += `\n${stats.join(" • ")}\n`;
|
|
144
141
|
}
|
|
145
142
|
|
|
@@ -218,17 +215,7 @@ export const handleBluesky: SpecialHandler = async (
|
|
|
218
215
|
}
|
|
219
216
|
}
|
|
220
217
|
|
|
221
|
-
|
|
222
|
-
return {
|
|
223
|
-
url,
|
|
224
|
-
finalUrl: url,
|
|
225
|
-
contentType: "text/markdown",
|
|
226
|
-
method: "bluesky-api",
|
|
227
|
-
content: output.content,
|
|
228
|
-
fetchedAt,
|
|
229
|
-
truncated: output.truncated,
|
|
230
|
-
notes: [`AT URI: ${atUri}`],
|
|
231
|
-
};
|
|
218
|
+
return buildResult(md, { url, method: "bluesky-api", fetchedAt, notes: [`AT URI: ${atUri}`] });
|
|
232
219
|
}
|
|
233
220
|
|
|
234
221
|
// Profile only
|
|
@@ -251,9 +238,9 @@ export const handleBluesky: SpecialHandler = async (
|
|
|
251
238
|
}
|
|
252
239
|
|
|
253
240
|
md += "---\n\n";
|
|
254
|
-
md += `- **Followers:** ${
|
|
255
|
-
md += `- **Following:** ${
|
|
256
|
-
md += `- **Posts:** ${
|
|
241
|
+
md += `- **Followers:** ${formatNumber(profile.followersCount || 0)}\n`;
|
|
242
|
+
md += `- **Following:** ${formatNumber(profile.followsCount || 0)}\n`;
|
|
243
|
+
md += `- **Posts:** ${formatNumber(profile.postsCount || 0)}\n`;
|
|
257
244
|
|
|
258
245
|
if (profile.createdAt) {
|
|
259
246
|
const joined = new Date(profile.createdAt).toLocaleDateString("en-US", {
|
|
@@ -266,17 +253,7 @@ export const handleBluesky: SpecialHandler = async (
|
|
|
266
253
|
|
|
267
254
|
md += `\n**DID:** \`${profile.did}\`\n`;
|
|
268
255
|
|
|
269
|
-
|
|
270
|
-
return {
|
|
271
|
-
url,
|
|
272
|
-
finalUrl: url,
|
|
273
|
-
contentType: "text/markdown",
|
|
274
|
-
method: "bluesky-api",
|
|
275
|
-
content: output.content,
|
|
276
|
-
fetchedAt,
|
|
277
|
-
truncated: output.truncated,
|
|
278
|
-
notes: ["Fetched via AT Protocol API"],
|
|
279
|
-
};
|
|
256
|
+
return buildResult(md, { url, method: "bluesky-api", fetchedAt, notes: ["Fetched via AT Protocol API"] });
|
|
280
257
|
}
|
|
281
258
|
} catch {}
|
|
282
259
|
|
package/src/web/scrapers/brew.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatNumber, loadPage } from "./types";
|
|
3
3
|
|
|
4
4
|
interface BrewFormula {
|
|
5
5
|
name: string;
|
|
@@ -97,7 +97,7 @@ export const handleBrew: SpecialHandler = async (
|
|
|
97
97
|
|
|
98
98
|
const installs = getInstallCount(formula.analytics);
|
|
99
99
|
if (installs !== null) {
|
|
100
|
-
md += `**Installs (30d):** ${
|
|
100
|
+
md += `**Installs (30d):** ${formatNumber(installs)}\n`;
|
|
101
101
|
}
|
|
102
102
|
md += "\n";
|
|
103
103
|
|
|
@@ -140,7 +140,7 @@ export const handleBrew: SpecialHandler = async (
|
|
|
140
140
|
|
|
141
141
|
const installs = getInstallCount(cask.analytics);
|
|
142
142
|
if (installs !== null) {
|
|
143
|
-
md += `**Installs (30d):** ${
|
|
143
|
+
md += `**Installs (30d):** ${formatNumber(installs)}\n`;
|
|
144
144
|
}
|
|
145
145
|
md += "\n";
|
|
146
146
|
|
|
@@ -160,17 +160,12 @@ export const handleBrew: SpecialHandler = async (
|
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
-
|
|
164
|
-
return {
|
|
163
|
+
return buildResult(md, {
|
|
165
164
|
url,
|
|
166
|
-
finalUrl: url,
|
|
167
|
-
contentType: "text/markdown",
|
|
168
165
|
method: "brew",
|
|
169
|
-
content: output.content,
|
|
170
166
|
fetchedAt,
|
|
171
|
-
truncated: output.truncated,
|
|
172
167
|
notes: [`Fetched via Homebrew ${isFormula ? "formula" : "cask"} API`],
|
|
173
|
-
};
|
|
168
|
+
});
|
|
174
169
|
} catch {}
|
|
175
170
|
|
|
176
171
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, loadPage } from "./types";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handle cheat.sh / cht.sh URLs for command cheatsheets
|
|
@@ -61,17 +61,7 @@ export const handleCheatSh: SpecialHandler = async (
|
|
|
61
61
|
md += `\`\`\`\n${content}\n\`\`\`\n`;
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
return {
|
|
66
|
-
url,
|
|
67
|
-
finalUrl: url,
|
|
68
|
-
contentType: "text/markdown",
|
|
69
|
-
method: "cheat.sh",
|
|
70
|
-
content: output.content,
|
|
71
|
-
fetchedAt,
|
|
72
|
-
truncated: output.truncated,
|
|
73
|
-
notes: ["Fetched via cheat.sh"],
|
|
74
|
-
};
|
|
64
|
+
return buildResult(md, { url, method: "cheat.sh", fetchedAt, notes: ["Fetched via cheat.sh"] });
|
|
75
65
|
} catch {}
|
|
76
66
|
|
|
77
67
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, formatIsoDate, formatNumber, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface NuGetODataEntry {
|
|
5
5
|
Id: string;
|
|
@@ -25,6 +25,13 @@ interface NuGetODataResponse {
|
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
function extractXmlField(xml: string, fieldName: string): string | null {
|
|
29
|
+
const pattern = new RegExp(`<d:${fieldName}[^>]*>([\\s\\S]*?)</d:${fieldName}>`, "i");
|
|
30
|
+
const match = xml.match(pattern);
|
|
31
|
+
if (!match) return null;
|
|
32
|
+
return match[1].trim();
|
|
33
|
+
}
|
|
34
|
+
|
|
28
35
|
/**
|
|
29
36
|
* Handle Chocolatey package URLs via NuGet v2 OData API
|
|
30
37
|
*/
|
|
@@ -59,21 +66,61 @@ export const handleChocolatey: SpecialHandler = async (
|
|
|
59
66
|
timeout,
|
|
60
67
|
signal,
|
|
61
68
|
headers: {
|
|
62
|
-
Accept: "application/
|
|
69
|
+
Accept: "application/atom+xml, application/xml",
|
|
63
70
|
},
|
|
64
71
|
});
|
|
65
72
|
|
|
66
|
-
if (!result.ok)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
+
if (!result.ok) {
|
|
74
|
+
const fallback = `# ${packageName}\n\nChocolatey package metadata is currently unavailable.\n\n---\n**Install:** \`choco install ${packageName}\`\n`;
|
|
75
|
+
return buildResult(fallback, {
|
|
76
|
+
url,
|
|
77
|
+
method: "chocolatey",
|
|
78
|
+
fetchedAt,
|
|
79
|
+
notes: ["Chocolatey API request failed"],
|
|
80
|
+
});
|
|
73
81
|
}
|
|
74
82
|
|
|
75
|
-
|
|
76
|
-
|
|
83
|
+
let pkg = (() => {
|
|
84
|
+
const data = tryParseJson<NuGetODataResponse>(result.content);
|
|
85
|
+
return data?.d?.results?.[0] ?? null;
|
|
86
|
+
})();
|
|
87
|
+
|
|
88
|
+
if (!pkg) {
|
|
89
|
+
const xmlId = extractXmlField(result.content, "Id");
|
|
90
|
+
if (!xmlId) {
|
|
91
|
+
const fallback = `# ${packageName}\n\nChocolatey package metadata could not be parsed.\n\n---\n**Install:** \`choco install ${packageName}\`\n`;
|
|
92
|
+
return buildResult(fallback, {
|
|
93
|
+
url,
|
|
94
|
+
method: "chocolatey",
|
|
95
|
+
fetchedAt,
|
|
96
|
+
notes: ["Chocolatey API response parsing failed"],
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
pkg = {
|
|
101
|
+
Id: xmlId,
|
|
102
|
+
Version: extractXmlField(result.content, "Version") || "",
|
|
103
|
+
Title: extractXmlField(result.content, "Title") || undefined,
|
|
104
|
+
Description: extractXmlField(result.content, "Description") || undefined,
|
|
105
|
+
Summary: extractXmlField(result.content, "Summary") || undefined,
|
|
106
|
+
Authors: extractXmlField(result.content, "Authors") || undefined,
|
|
107
|
+
ProjectUrl: extractXmlField(result.content, "ProjectUrl") || undefined,
|
|
108
|
+
PackageSourceUrl: extractXmlField(result.content, "PackageSourceUrl") || undefined,
|
|
109
|
+
Tags: extractXmlField(result.content, "Tags") || undefined,
|
|
110
|
+
DownloadCount: (() => {
|
|
111
|
+
const value = extractXmlField(result.content, "DownloadCount");
|
|
112
|
+
return value ? Number.parseInt(value, 10) : undefined;
|
|
113
|
+
})(),
|
|
114
|
+
VersionDownloadCount: (() => {
|
|
115
|
+
const value = extractXmlField(result.content, "VersionDownloadCount");
|
|
116
|
+
return value ? Number.parseInt(value, 10) : undefined;
|
|
117
|
+
})(),
|
|
118
|
+
Published: extractXmlField(result.content, "Published") || undefined,
|
|
119
|
+
LicenseUrl: extractXmlField(result.content, "LicenseUrl") || undefined,
|
|
120
|
+
ReleaseNotes: extractXmlField(result.content, "ReleaseNotes") || undefined,
|
|
121
|
+
Dependencies: extractXmlField(result.content, "Dependencies") || undefined,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
77
124
|
|
|
78
125
|
// Build markdown output
|
|
79
126
|
let md = `# ${pkg.Title || pkg.Id}\n\n`;
|
|
@@ -91,16 +138,16 @@ export const handleChocolatey: SpecialHandler = async (
|
|
|
91
138
|
md += "\n";
|
|
92
139
|
|
|
93
140
|
if (pkg.DownloadCount !== undefined) {
|
|
94
|
-
md += `**Total Downloads:** ${
|
|
141
|
+
md += `**Total Downloads:** ${formatNumber(pkg.DownloadCount)}`;
|
|
95
142
|
if (pkg.VersionDownloadCount !== undefined) {
|
|
96
|
-
md += ` · **Version Downloads:** ${
|
|
143
|
+
md += ` · **Version Downloads:** ${formatNumber(pkg.VersionDownloadCount)}`;
|
|
97
144
|
}
|
|
98
145
|
md += "\n";
|
|
99
146
|
}
|
|
100
147
|
|
|
101
148
|
if (pkg.Published) {
|
|
102
|
-
const
|
|
103
|
-
md += `**Published:** ${
|
|
149
|
+
const published = formatIsoDate(pkg.Published);
|
|
150
|
+
if (published) md += `**Published:** ${published}\n`;
|
|
104
151
|
}
|
|
105
152
|
|
|
106
153
|
md += "\n";
|
|
@@ -141,17 +188,7 @@ export const handleChocolatey: SpecialHandler = async (
|
|
|
141
188
|
|
|
142
189
|
md += `\n---\n**Install:** \`choco install ${packageName}\`\n`;
|
|
143
190
|
|
|
144
|
-
|
|
145
|
-
return {
|
|
146
|
-
url,
|
|
147
|
-
finalUrl: url,
|
|
148
|
-
contentType: "text/markdown",
|
|
149
|
-
method: "chocolatey",
|
|
150
|
-
content: output.content,
|
|
151
|
-
fetchedAt,
|
|
152
|
-
truncated: output.truncated,
|
|
153
|
-
notes: ["Fetched via Chocolatey NuGet API"],
|
|
154
|
-
};
|
|
191
|
+
return buildResult(md, { url, method: "chocolatey", fetchedAt, notes: ["Fetched via Chocolatey NuGet API"] });
|
|
155
192
|
} catch {}
|
|
156
193
|
|
|
157
194
|
return null;
|
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
import { parseFrontmatter } from "../../utils/frontmatter";
|
|
2
2
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
3
|
-
import {
|
|
3
|
+
import { buildResult, loadPage } from "./types";
|
|
4
|
+
import { asString } from "./utils";
|
|
4
5
|
|
|
5
6
|
const ALLOWED_HOSTS = new Set(["choosealicense.com", "www.choosealicense.com"]);
|
|
6
7
|
const LICENSE_PATH = /^\/licenses\/([^/]+)\/?$/i;
|
|
7
8
|
const APPENDIX_PATH = /^\/appendix\/?$/i;
|
|
8
9
|
|
|
9
|
-
function asString(value: unknown): string | undefined {
|
|
10
|
-
if (typeof value !== "string") return undefined;
|
|
11
|
-
const trimmed = value.trim();
|
|
12
|
-
return trimmed.length > 0 ? trimmed : undefined;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
10
|
function normalizeList(value: unknown): string[] {
|
|
16
11
|
if (Array.isArray(value)) {
|
|
17
12
|
return value
|
|
@@ -93,17 +88,7 @@ export const handleChooseALicense: SpecialHandler = async (
|
|
|
93
88
|
md += `---\n\n## License Text\n\n${licenseText}\n`;
|
|
94
89
|
}
|
|
95
90
|
|
|
96
|
-
|
|
97
|
-
return {
|
|
98
|
-
url,
|
|
99
|
-
finalUrl: url,
|
|
100
|
-
contentType: "text/markdown",
|
|
101
|
-
method: "choosealicense",
|
|
102
|
-
content: output.content,
|
|
103
|
-
fetchedAt,
|
|
104
|
-
truncated: output.truncated,
|
|
105
|
-
notes: ["Fetched via Choose a License"],
|
|
106
|
-
};
|
|
91
|
+
return buildResult(md, { url, method: "choosealicense", fetchedAt, notes: ["Fetched via Choose a License"] });
|
|
107
92
|
} catch {}
|
|
108
93
|
|
|
109
94
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RenderResult, SpecialHandler } from "./types";
|
|
2
|
-
import {
|
|
2
|
+
import { buildResult, loadPage, tryParseJson } from "./types";
|
|
3
3
|
|
|
4
4
|
interface KevEntry {
|
|
5
5
|
cveID: string;
|
|
@@ -53,12 +53,8 @@ export const handleCisaKev: SpecialHandler = async (
|
|
|
53
53
|
|
|
54
54
|
if (!result.ok) return null;
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
data = JSON.parse(result.content) as KevCatalog;
|
|
59
|
-
} catch {
|
|
60
|
-
return null;
|
|
61
|
-
}
|
|
56
|
+
const data = tryParseJson<KevCatalog>(result.content);
|
|
57
|
+
if (!data) return null;
|
|
62
58
|
|
|
63
59
|
const entry = data.vulnerabilities?.find(item => item.cveID?.toUpperCase() === cveId);
|
|
64
60
|
if (!entry) return null;
|
|
@@ -83,17 +79,7 @@ export const handleCisaKev: SpecialHandler = async (
|
|
|
83
79
|
md += `## Required Action\n\n${entry.requiredAction}\n\n`;
|
|
84
80
|
}
|
|
85
81
|
|
|
86
|
-
|
|
87
|
-
return {
|
|
88
|
-
url,
|
|
89
|
-
finalUrl: url,
|
|
90
|
-
contentType: "text/markdown",
|
|
91
|
-
method: "cisa-kev",
|
|
92
|
-
content: output.content,
|
|
93
|
-
fetchedAt,
|
|
94
|
-
truncated: output.truncated,
|
|
95
|
-
notes: ["Fetched via CISA KEV feed"],
|
|
96
|
-
};
|
|
82
|
+
return buildResult(md, { url, method: "cisa-kev", fetchedAt, notes: ["Fetched via CISA KEV feed"] });
|
|
97
83
|
} catch {}
|
|
98
84
|
|
|
99
85
|
return null;
|