@4da/mcp-server 6.0.2 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +433 -0
  2. package/README.md +26 -18
  3. package/dist/db.d.ts +73 -8
  4. package/dist/db.js +180 -43
  5. package/dist/doctor.d.ts +1 -1
  6. package/dist/doctor.js +15 -9
  7. package/dist/embeddings.js +19 -7
  8. package/dist/gitignore.js +9 -3
  9. package/dist/index.js +43 -4
  10. package/dist/live/cache.d.ts +31 -2
  11. package/dist/live/cache.js +113 -1
  12. package/dist/live/changelog-classify.d.ts +9 -3
  13. package/dist/live/changelog-classify.js +125 -12
  14. package/dist/live/changelog.d.ts +22 -3
  15. package/dist/live/changelog.js +132 -38
  16. package/dist/live/dev-scope.d.ts +2 -2
  17. package/dist/live/fix-paths.d.ts +118 -0
  18. package/dist/live/fix-paths.js +498 -0
  19. package/dist/live/github-releases.d.ts +69 -0
  20. package/dist/live/github-releases.js +252 -0
  21. package/dist/live/go-registry.d.ts +5 -0
  22. package/dist/live/go-registry.js +1 -1
  23. package/dist/live/go-releases.d.ts +48 -0
  24. package/dist/live/go-releases.js +196 -0
  25. package/dist/live/index.d.ts +34 -8
  26. package/dist/live/index.js +66 -7
  27. package/dist/live/lockfile-parents.d.ts +61 -0
  28. package/dist/live/lockfile-parents.js +360 -0
  29. package/dist/live/lockfile-parsers-pygo.d.ts +31 -0
  30. package/dist/live/lockfile-parsers-pygo.js +178 -28
  31. package/dist/live/lockfile-parsers.d.ts +4 -1
  32. package/dist/live/lockfile-parsers.js +57 -24
  33. package/dist/live/lockfile-types.d.ts +12 -3
  34. package/dist/live/lockfile-types.js +2 -1
  35. package/dist/live/osv-ranges.d.ts +85 -0
  36. package/dist/live/osv-ranges.js +136 -0
  37. package/dist/live/osv-scanner.d.ts +1 -0
  38. package/dist/live/osv-scanner.js +85 -78
  39. package/dist/live/package-archive.d.ts +2 -0
  40. package/dist/live/package-archive.js +7 -4
  41. package/dist/live/pypi-releases.d.ts +44 -0
  42. package/dist/live/pypi-releases.js +103 -0
  43. package/dist/live/range-satisfies.d.ts +21 -0
  44. package/dist/live/range-satisfies.js +190 -0
  45. package/dist/live/rate-limiter.d.ts +6 -0
  46. package/dist/live/rate-limiter.js +5 -0
  47. package/dist/live/registry-health.js +2 -0
  48. package/dist/live/release-metadata.d.ts +17 -8
  49. package/dist/live/release-metadata.js +6 -3
  50. package/dist/live/resolution.d.ts +11 -1
  51. package/dist/live/resolution.js +16 -6
  52. package/dist/live/semver-utils.d.ts +8 -0
  53. package/dist/live/semver-utils.js +10 -2
  54. package/dist/live/types.d.ts +17 -1
  55. package/dist/live/upgrade-go.d.ts +62 -0
  56. package/dist/live/upgrade-go.js +130 -0
  57. package/dist/live/upgrade-pypi.d.ts +42 -0
  58. package/dist/live/upgrade-pypi.js +85 -0
  59. package/dist/live/upgrade-sources.d.ts +30 -11
  60. package/dist/live/upgrade-sources.js +93 -21
  61. package/dist/live/zip-archive.d.ts +22 -0
  62. package/dist/live/zip-archive.js +98 -0
  63. package/dist/memory-store.d.ts +19 -0
  64. package/dist/memory-store.js +8 -0
  65. package/dist/native-bindings.d.ts +7 -2
  66. package/dist/native-bindings.js +4 -0
  67. package/dist/project-tree.d.ts +50 -8
  68. package/dist/project-tree.js +121 -29
  69. package/dist/schema-registry.js +5 -5
  70. package/dist/server-instructions.d.ts +6 -0
  71. package/dist/server-instructions.js +20 -15
  72. package/dist/sqlite-driver.d.ts +134 -0
  73. package/dist/sqlite-driver.js +359 -0
  74. package/dist/tool-dispatch.d.ts +8 -1
  75. package/dist/tool-dispatch.js +17 -2
  76. package/dist/tools/agent-memory.js +10 -2
  77. package/dist/tools/app-plan.d.ts +11 -1
  78. package/dist/tools/app-plan.js +32 -6
  79. package/dist/tools/briefing-task-scope.d.ts +21 -2
  80. package/dist/tools/briefing-task-scope.js +97 -4
  81. package/dist/tools/decision-memory.js +9 -3
  82. package/dist/tools/dependency-check-signals.d.ts +23 -5
  83. package/dist/tools/dependency-check-signals.js +62 -19
  84. package/dist/tools/dependency-check-versions.d.ts +94 -0
  85. package/dist/tools/dependency-check-versions.js +213 -0
  86. package/dist/tools/dependency-check.d.ts +27 -11
  87. package/dist/tools/dependency-check.js +321 -74
  88. package/dist/tools/dependency-health.d.ts +1 -1
  89. package/dist/tools/dependency-health.js +5 -0
  90. package/dist/tools/knowledge-gap-ranges.d.ts +1 -1
  91. package/dist/tools/knowledge-gap-ranges.js +1 -1
  92. package/dist/tools/knowledge-gap-scope.d.ts +2 -2
  93. package/dist/tools/project-scope.d.ts +45 -0
  94. package/dist/tools/project-scope.js +119 -0
  95. package/dist/tools/record-feedback.js +1 -1
  96. package/dist/tools/upgrade-impact-callsites.d.ts +1 -0
  97. package/dist/tools/upgrade-impact-callsites.js +23 -5
  98. package/dist/tools/upgrade-impact-changelog.d.ts +41 -0
  99. package/dist/tools/upgrade-impact-changelog.js +79 -0
  100. package/dist/tools/upgrade-impact-imports.d.ts +42 -0
  101. package/dist/tools/upgrade-impact-imports.js +260 -0
  102. package/dist/tools/upgrade-impact-literals.js +1 -1
  103. package/dist/tools/upgrade-impact-report.d.ts +23 -0
  104. package/dist/tools/upgrade-impact-report.js +45 -6
  105. package/dist/tools/upgrade-impact.d.ts +13 -4
  106. package/dist/tools/upgrade-impact.js +236 -85
  107. package/dist/tools/upgrade-planner-fixpaths.d.ts +40 -0
  108. package/dist/tools/upgrade-planner-fixpaths.js +0 -0
  109. package/dist/tools/upgrade-planner.d.ts +31 -3
  110. package/dist/tools/upgrade-planner.js +74 -19
  111. package/dist/tools/vulnerability-scan-concise.d.ts +13 -1
  112. package/dist/tools/vulnerability-scan-concise.js +15 -3
  113. package/dist/tools/vulnerability-scan-format.d.ts +58 -1
  114. package/dist/tools/vulnerability-scan-format.js +107 -11
  115. package/dist/tools/vulnerability-scan.d.ts +104 -13
  116. package/dist/tools/vulnerability-scan.js +27 -3
  117. package/dist/tools/what-should-i-know.d.ts +22 -2
  118. package/dist/tools/what-should-i-know.js +122 -12
  119. package/package.json +10 -32
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Which project a planning or briefing answer is about.
3
+ *
4
+ * Measured 2026-10-07: upgrade_planner called from D:\4DA returned the 4DA
5
+ * app's cross-project plan, and its top steps were for navcal, verax and
6
+ * 4da-ledger. upgrade_planner and what_should_i_know now take `project_path`:
7
+ * omitted, it is the project the server was started for (FOURDA_PROJECT_DIR,
8
+ * else the working directory) when that directory is a project; `"*"` asks
9
+ * for every project.
10
+ */
11
+ import type { FourDADatabase } from "../db.js";
12
+ import { LiveIntelligence } from "../live/index.js";
13
+ export declare const ALL_PROJECTS = "*";
14
+ /** The project the server was started for: FOURDA_PROJECT_DIR (an unsubstituted "${...}" template is ignored), else cwd. */
15
+ export declare function serverProjectDir(): string;
16
+ /** True when `dir` holds a manifest or lockfile of a supported ecosystem. */
17
+ export declare function isProjectDir(dir: string): boolean;
18
+ /** Lowercased, forward slashes, no trailing slash: the form the app's plan stores project paths in. */
19
+ export declare function normalizeProjectPath(p: string): string;
20
+ /** True when `candidate` is `root` or inside it. */
21
+ export declare function isWithin(candidate: string, root: string): boolean;
22
+ export type ProjectScope = {
23
+ kind: "project";
24
+ path: string;
25
+ source: "argument" | "server_project";
26
+ } | {
27
+ kind: "all";
28
+ reason: string;
29
+ } | {
30
+ kind: "error";
31
+ error: string;
32
+ };
33
+ /**
34
+ * The scope for one call. An explicit path must be a directory on this
35
+ * machine; `"*"` is every project; omitted, the server's project when it is
36
+ * one, otherwise every project (a server started in a home directory has no
37
+ * project to narrow to).
38
+ */
39
+ export declare function resolveProjectScope(projectPath: string | undefined): ProjectScope;
40
+ /**
41
+ * The live layer for `scope`: the server's own when it already covers that
42
+ * root, otherwise a fresh one resolved from the project tree at that path (as
43
+ * vulnerability_scan does for its `project_path`).
44
+ */
45
+ export declare function liveIntelFor<T extends Pick<LiveIntelligence, "getProjectRoot">>(liveIntel: T | null, db: FourDADatabase | null, scope: ProjectScope): T | LiveIntelligence | null;
@@ -0,0 +1,119 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * Which project a planning or briefing answer is about.
4
+ *
5
+ * Measured 2026-10-07: upgrade_planner called from D:\4DA returned the 4DA
6
+ * app's cross-project plan, and its top steps were for navcal, verax and
7
+ * 4da-ledger. upgrade_planner and what_should_i_know now take `project_path`:
8
+ * omitted, it is the project the server was started for (FOURDA_PROJECT_DIR,
9
+ * else the working directory) when that directory is a project; `"*"` asks
10
+ * for every project.
11
+ */
12
+ import * as fs from "node:fs";
13
+ import * as path from "node:path";
14
+ import { getServerCacheDb } from "../live/cache.js";
15
+ import { LiveIntelligence } from "../live/index.js";
16
+ import { scanProjectTree, treeResolutionGroups } from "../project-tree.js";
17
+ /** Files that make a directory a project (manifests and lockfiles of the supported ecosystems). */
18
+ const PROJECT_MARKERS = [
19
+ "package.json",
20
+ "Cargo.toml",
21
+ "pyproject.toml",
22
+ "setup.py",
23
+ "requirements.txt",
24
+ "Pipfile",
25
+ "go.mod",
26
+ "pnpm-lock.yaml",
27
+ "package-lock.json",
28
+ "yarn.lock",
29
+ "bun.lock",
30
+ "Cargo.lock",
31
+ "poetry.lock",
32
+ "uv.lock",
33
+ ];
34
+ export const ALL_PROJECTS = "*";
35
+ /** The project the server was started for: FOURDA_PROJECT_DIR (an unsubstituted "${...}" template is ignored), else cwd. */
36
+ export function serverProjectDir() {
37
+ const dir = process.env.FOURDA_PROJECT_DIR?.trim();
38
+ if (dir && !dir.includes("${"))
39
+ return dir;
40
+ return process.cwd();
41
+ }
42
+ /** True when `dir` holds a manifest or lockfile of a supported ecosystem. */
43
+ export function isProjectDir(dir) {
44
+ return PROJECT_MARKERS.some((m) => {
45
+ try {
46
+ return fs.statSync(path.join(dir, m)).isFile();
47
+ }
48
+ catch {
49
+ return false;
50
+ }
51
+ });
52
+ }
53
+ /** Lowercased, forward slashes, no trailing slash: the form the app's plan stores project paths in. */
54
+ export function normalizeProjectPath(p) {
55
+ return path.resolve(p).replace(/\\/g, "/").toLowerCase().replace(/\/+$/, "");
56
+ }
57
+ /** True when `candidate` is `root` or inside it. */
58
+ export function isWithin(candidate, root) {
59
+ const c = normalizeProjectPath(candidate);
60
+ const r = normalizeProjectPath(root);
61
+ return c === r || c.startsWith(`${r}/`);
62
+ }
63
+ /**
64
+ * The scope for one call. An explicit path must be a directory on this
65
+ * machine; `"*"` is every project; omitted, the server's project when it is
66
+ * one, otherwise every project (a server started in a home directory has no
67
+ * project to narrow to).
68
+ */
69
+ export function resolveProjectScope(projectPath) {
70
+ const given = projectPath?.trim();
71
+ if (given === ALL_PROJECTS)
72
+ return { kind: "all", reason: 'project_path "*" asks for every project' };
73
+ if (given) {
74
+ const abs = path.resolve(given);
75
+ let isDir = false;
76
+ try {
77
+ isDir = fs.statSync(abs).isDirectory();
78
+ }
79
+ catch {
80
+ isDir = false;
81
+ }
82
+ if (!isDir) {
83
+ return {
84
+ kind: "error",
85
+ error: `project_path "${given}" is not a directory on this machine. Pass the absolute path of a project root, "*" for every project, or omit it for ${serverProjectDir()}.`,
86
+ };
87
+ }
88
+ return { kind: "project", path: abs, source: "argument" };
89
+ }
90
+ const own = serverProjectDir();
91
+ if (isProjectDir(own))
92
+ return { kind: "project", path: path.resolve(own), source: "server_project" };
93
+ return { kind: "all", reason: `the server's directory ${own} is not a project (no manifest or lockfile)` };
94
+ }
95
+ /**
96
+ * The live layer for `scope`: the server's own when it already covers that
97
+ * root, otherwise a fresh one resolved from the project tree at that path (as
98
+ * vulnerability_scan does for its `project_path`).
99
+ */
100
+ export function liveIntelFor(liveIntel, db, scope) {
101
+ if (scope.kind !== "project" || !liveIntel)
102
+ return liveIntel;
103
+ const root = liveIntel.getProjectRoot();
104
+ if (root && normalizeProjectPath(root) === normalizeProjectPath(scope.path))
105
+ return liveIntel;
106
+ // A scope the server did not start for: resolve that tree. The connection
107
+ // is only used for its cache (LiveCache never writes to the app's database).
108
+ let raw;
109
+ try {
110
+ raw = db?.getRawDb();
111
+ }
112
+ catch {
113
+ raw = undefined;
114
+ }
115
+ const scoped = new LiveIntelligence(raw ?? getServerCacheDb());
116
+ const { entries, discovery } = scanProjectTree(scope.path);
117
+ scoped.initFromProjectTree(scope.path, treeResolutionGroups(entries), discovery);
118
+ return scoped;
119
+ }
@@ -17,7 +17,7 @@ Feedback actions:
17
17
  - "dismiss": User dismissed the item
18
18
  - "mark_irrelevant": User said the item is not relevant (also records a relevance label)
19
19
 
20
- Only "save" and "mark_irrelevant" are relevance labels; they are what 4DA's accuracy measurement reads. Use them only when the user actually says whether the item matters to them. This does not train content preferences.`,
20
+ Only "save" and "mark_irrelevant" are relevance labels. Use them only when the user actually says whether the item matters to them. This does not train content preferences. Recorded in the 4DA desktop app's own interactions/feedback tables.`,
21
21
  inputSchema: {
22
22
  type: "object",
23
23
  properties: {
@@ -39,6 +39,7 @@ export declare function scanRustSource(source: string, crateName: string, symbol
39
39
  export declare function scanCallSites(root: string, pkg: string, ecosystem: UpgradeEcosystem, opts?: {
40
40
  timeBudgetMs?: number;
41
41
  concurrency?: number;
42
+ importNames?: string[];
42
43
  }): Promise<CallSiteReport>;
43
44
  /**
44
45
  * Symbols (≥ 3 chars) that `text` names at a word boundary, case-sensitively.
@@ -18,12 +18,16 @@
18
18
  import * as fs from "node:fs";
19
19
  import * as path from "node:path";
20
20
  import { IgnoreRules } from "../gitignore.js";
21
+ import { scanGoSource, scanPythonSource } from "./upgrade-impact-imports.js";
21
22
  const SKIP_DIRS = new Set([
22
23
  "node_modules", "target", ".git", "dist", "build", "out", ".next",
23
- "coverage", "vendor", ".claude", ".venv",
24
+ "coverage", "vendor", ".claude", ".venv", "venv", "__pycache__", "site-packages",
24
25
  ]);
25
26
  const NPM_EXTS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts", ".vue", ".svelte"]);
26
27
  const RUST_EXTS = new Set([".rs"]);
28
+ const PY_EXTS = new Set([".py", ".pyi"]);
29
+ const GO_EXTS = new Set([".go"]);
30
+ const EXTS = { npm: NPM_EXTS, "crates.io": RUST_EXTS, PyPI: PY_EXTS, Go: GO_EXTS };
27
31
  const MAX_FILES = 6000;
28
32
  const MAX_FILE_BYTES = 1024 * 1024;
29
33
  const MAX_REPORTED_FILES = 50;
@@ -157,11 +161,25 @@ function listSources(root, exts, budget) {
157
161
  * all, and a scan that runs out of time says its counts are lower bounds.
158
162
  */
159
163
  export async function scanCallSites(root, pkg, ecosystem, opts = {}) {
160
- const exts = ecosystem === "npm" ? NPM_EXTS : RUST_EXTS;
164
+ const exts = EXTS[ecosystem];
161
165
  const budget = { left: MAX_FILES };
162
166
  const symbols = new Set();
163
167
  const hits = [];
164
- const needle = ecosystem === "npm" ? pkg : pkg.replace(/-/g, "_");
168
+ // PyPI: the module names the distribution is imported as (upgrade-impact-imports.ts).
169
+ const importNames = opts.importNames ?? [pkg];
170
+ const needles = ecosystem === "PyPI" ? importNames : [ecosystem === "crates.io" ? pkg.replace(/-/g, "_") : pkg];
171
+ const scan = (source) => {
172
+ switch (ecosystem) {
173
+ case "npm":
174
+ return scanNpmSource(source, pkg, symbols);
175
+ case "crates.io":
176
+ return scanRustSource(source, pkg, symbols);
177
+ case "PyPI":
178
+ return scanPythonSource(source, importNames, symbols);
179
+ case "Go":
180
+ return scanGoSource(source, pkg, symbols);
181
+ }
182
+ };
165
183
  const deadline = Date.now() + (opts.timeBudgetMs ?? CALLSITE_TIME_BUDGET_MS);
166
184
  const files = listSources(root, exts, budget);
167
185
  let next = 0;
@@ -183,9 +201,9 @@ export async function scanCallSites(root, pkg, ecosystem, opts = {}) {
183
201
  catch {
184
202
  continue;
185
203
  }
186
- if (!source.includes(needle))
204
+ if (!needles.some((n) => source.includes(n)))
187
205
  continue;
188
- const matches = ecosystem === "npm" ? scanNpmSource(source, pkg, symbols) : scanRustSource(source, pkg, symbols);
206
+ const matches = scan(source);
189
207
  if (matches > 0)
190
208
  hits.push({ path: path.relative(root, file).split(path.sep).join("/"), matches });
191
209
  }
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Which changelog `upgrade_impact` reports, and what it says when there is none.
3
+ *
4
+ * Order: the changelog inside the target version's registry archive (always;
5
+ * Decision 1 option A), then, only when the user opted in with
6
+ * FOURDA_GITHUB_CHANGELOG and the archive does not cover the range, the
7
+ * GitHub fallback (a changelog file the project links, then its releases;
8
+ * Decision 1 option B). The GitHub result replaces the archive's only when it
9
+ * covers the range or has more of its releases.
10
+ *
11
+ * With the fallback off and no usable changelog, the answer says the
12
+ * changelog is unavailable and exactly how to enable the fallback, or that
13
+ * enabling it would not help (no GitHub repository). It never fills the gap
14
+ * with a guess.
15
+ */
16
+ import { type ChangelogSection, type VersionOrder } from "../live/changelog.js";
17
+ import { type ChangelogResult, type PublishedVersion, type RegistryIndex, type UpgradeNet } from "../live/upgrade-sources.js";
18
+ export interface ResolvedChangelog {
19
+ changelog: ChangelogResult;
20
+ range: {
21
+ sections: ChangelogSection[];
22
+ coversRange: boolean;
23
+ };
24
+ /** Present when the archive did not cover the range: what the fallback did, or how to turn it on. */
25
+ github_fallback?: Record<string, unknown>;
26
+ }
27
+ export interface ChangelogRequest {
28
+ net: UpgradeNet;
29
+ index: RegistryIndex;
30
+ target: PublishedVersion;
31
+ from: string;
32
+ to: string;
33
+ firstInRange: string;
34
+ order: VersionOrder;
35
+ githubEnabled: boolean;
36
+ /** Repository to read releases from (Go: the proxy's VCS origin when known). */
37
+ repository: string | null;
38
+ /** Go: the module's tag prefix in its repository. */
39
+ tagPrefix?: string | null;
40
+ }
41
+ export declare function resolveChangelog(req: ChangelogRequest): Promise<ResolvedChangelog>;
@@ -0,0 +1,79 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * Which changelog `upgrade_impact` reports, and what it says when there is none.
4
+ *
5
+ * Order: the changelog inside the target version's registry archive (always;
6
+ * Decision 1 option A), then, only when the user opted in with
7
+ * FOURDA_GITHUB_CHANGELOG and the archive does not cover the range, the
8
+ * GitHub fallback (a changelog file the project links, then its releases;
9
+ * Decision 1 option B). The GitHub result replaces the archive's only when it
10
+ * covers the range or has more of its releases.
11
+ *
12
+ * With the fallback off and no usable changelog, the answer says the
13
+ * changelog is unavailable and exactly how to enable the fallback, or that
14
+ * enabling it would not help (no GitHub repository). It never fills the gap
15
+ * with a guess.
16
+ */
17
+ import { selectRange } from "../live/changelog.js";
18
+ import { GITHUB_CHANGELOG_ENV, getGithubChangelog, parseGithubRepo } from "../live/github-releases.js";
19
+ import { getChangelog } from "../live/upgrade-sources.js";
20
+ function rangeOf(result, req) {
21
+ return result.found && result.sections
22
+ ? selectRange(result.sections, req.from, req.to, req.firstInRange, req.order)
23
+ : { sections: [], coversRange: false };
24
+ }
25
+ export async function resolveChangelog(req) {
26
+ const archive = await getChangelog(req.net, req.index, req.target);
27
+ const archiveRange = rangeOf(archive, req);
28
+ if (archiveRange.coversRange)
29
+ return { changelog: archive, range: archiveRange };
30
+ const repo = parseGithubRepo(req.repository);
31
+ const pyLinks = (req.index.changelogLinks ?? []).filter((l) => /^https:\/\/(?:github\.com|raw\.githubusercontent\.com)\//i.test(l));
32
+ const archiveState = !archive.found
33
+ ? archive.reason ?? "no changelog in the archive"
34
+ : archiveRange.sections.length === 0
35
+ ? `${archive.file} has no section for these releases`
36
+ : `${archive.file} covers only part of the range`;
37
+ if (!req.githubEnabled) {
38
+ const where = repo ? `github.com/${repo.owner}/${repo.repo}` : null;
39
+ return {
40
+ changelog: archive,
41
+ range: archiveRange,
42
+ github_fallback: where || pyLinks.length > 0
43
+ ? {
44
+ enabled: false,
45
+ how_to_enable: `Set ${GITHUB_CHANGELOG_ENV}=true in this MCP server's environment to read the release notes ${where ? `on ${where}` : "the project links on GitHub"} when the archive has none. GitHub then receives the repository name and the version range only; GITHUB_TOKEN, if set, raises its rate limit.`,
46
+ }
47
+ : {
48
+ enabled: false,
49
+ note: `The package names no GitHub repository, so ${GITHUB_CHANGELOG_ENV} would not find release notes either; see release_notes_url.`,
50
+ },
51
+ };
52
+ }
53
+ const gh = await getGithubChangelog(req.net, {
54
+ pkg: req.index.name,
55
+ ecosystem: req.index.ecosystem,
56
+ repository: req.repository,
57
+ changelogLinks: req.index.changelogLinks,
58
+ from: req.from,
59
+ to: req.to,
60
+ tagPrefix: req.tagPrefix ?? null,
61
+ order: req.order,
62
+ });
63
+ if (!gh.found) {
64
+ return { changelog: archive, range: archiveRange, github_fallback: { enabled: true, used: false, reason: gh.reason } };
65
+ }
66
+ const ghRange = rangeOf(gh, req);
67
+ if (ghRange.coversRange || ghRange.sections.length > archiveRange.sections.length) {
68
+ return {
69
+ changelog: gh,
70
+ range: ghRange,
71
+ github_fallback: { enabled: true, used: true, source: gh.source, url: gh.url, archive: archiveState },
72
+ };
73
+ }
74
+ return {
75
+ changelog: archive,
76
+ range: archiveRange,
77
+ github_fallback: { enabled: true, used: false, reason: "the GitHub release notes cover no more of the range than the archive's changelog" },
78
+ };
79
+ }
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Python and Go import sites for `upgrade_impact`.
3
+ *
4
+ * Go is exact: a file imports a module by its path (`"github.com/jackc/pgx/v5"`
5
+ * or a package below it), and the identifiers it reaches through the package
6
+ * name (`pgx.Connect`) are the symbols a changelog entry can name.
7
+ *
8
+ * Python is not: a file imports a MODULE (`import yaml`), while an upgrade names
9
+ * a DISTRIBUTION (`PyYAML`), and nothing in the source links the two. The
10
+ * mapping comes, in order of trust, from:
11
+ * 1. the installed distribution's own metadata in this project's virtualenv
12
+ * (`*.dist-info/top_level.txt`, else the top-level names in `RECORD`);
13
+ * 2. a short table of well-known distributions whose import name differs
14
+ * (beautifulsoup4 -> bs4, Pillow -> PIL, PyYAML -> yaml, ...);
15
+ * 3. the distribution name itself with `-` and `.` read as `_`, which is
16
+ * right for most packages and is reported as an ASSUMPTION every time,
17
+ * so a count of zero files is never mistaken for "not used".
18
+ */
19
+ export type ImportNameSource = "installed-metadata" | "known-mapping" | "assumed";
20
+ export interface PythonImportNames {
21
+ names: string[];
22
+ source: ImportNameSource;
23
+ /** Where the installed metadata was read, relative to the project root. */
24
+ evidence?: string;
25
+ }
26
+ /** The module name(s) a distribution is imported as, and how that was established. */
27
+ export declare function pythonImportNames(root: string | null, dist: string): PythonImportNames;
28
+ /**
29
+ * Count imports of any of `names` in one Python file and collect the names it
30
+ * reaches: `from x import A, B` gives A and B; `import x` / `import x as y`
31
+ * gives the attributes used as `x.attr` / `y.attr`.
32
+ */
33
+ export declare function scanPythonSource(source: string, names: string[], symbols: Set<string>): number;
34
+ /** True when import path `imp` is module `mod` or a package inside it, not a different major (`mod/v2`). */
35
+ export declare function goImportInModule(imp: string, mod: string): boolean;
36
+ /** The package name an import path is used through: its last element, skipping a `/vN` suffix and gopkg.in's `.vN`. */
37
+ export declare function goPackageName(imp: string): string;
38
+ /**
39
+ * Count imports of a Go module (or packages inside it) in one file and collect
40
+ * the exported identifiers reached through each import's package name.
41
+ */
42
+ export declare function scanGoSource(source: string, mod: string, symbols: Set<string>): number;
@@ -0,0 +1,260 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * Python and Go import sites for `upgrade_impact`.
4
+ *
5
+ * Go is exact: a file imports a module by its path (`"github.com/jackc/pgx/v5"`
6
+ * or a package below it), and the identifiers it reaches through the package
7
+ * name (`pgx.Connect`) are the symbols a changelog entry can name.
8
+ *
9
+ * Python is not: a file imports a MODULE (`import yaml`), while an upgrade names
10
+ * a DISTRIBUTION (`PyYAML`), and nothing in the source links the two. The
11
+ * mapping comes, in order of trust, from:
12
+ * 1. the installed distribution's own metadata in this project's virtualenv
13
+ * (`*.dist-info/top_level.txt`, else the top-level names in `RECORD`);
14
+ * 2. a short table of well-known distributions whose import name differs
15
+ * (beautifulsoup4 -> bs4, Pillow -> PIL, PyYAML -> yaml, ...);
16
+ * 3. the distribution name itself with `-` and `.` read as `_`, which is
17
+ * right for most packages and is reported as an ASSUMPTION every time,
18
+ * so a count of zero files is never mistaken for "not used".
19
+ */
20
+ import * as fs from "node:fs";
21
+ import * as path from "node:path";
22
+ import { normalizePypiName } from "../live/upgrade-pypi.js";
23
+ import { goBasePath, goPathMajor } from "../live/upgrade-go.js";
24
+ /**
25
+ * Distributions whose import name is not their normalised name. Each is the
26
+ * package's documented import (PyPI page or the project's own docs); a
27
+ * distribution that installs several top-level modules lists them all.
28
+ */
29
+ const KNOWN_IMPORT_NAMES = {
30
+ "beautifulsoup4": ["bs4"],
31
+ "pillow": ["PIL"],
32
+ "pyyaml": ["yaml"],
33
+ "scikit-learn": ["sklearn"],
34
+ "scikit-image": ["skimage"],
35
+ "opencv-python": ["cv2"],
36
+ "opencv-python-headless": ["cv2"],
37
+ "opencv-contrib-python": ["cv2"],
38
+ "opencv-contrib-python-headless": ["cv2"],
39
+ "python-dateutil": ["dateutil"],
40
+ "pyjwt": ["jwt"],
41
+ "python-dotenv": ["dotenv"],
42
+ "protobuf": ["google.protobuf"],
43
+ "attrs": ["attr", "attrs"],
44
+ "psycopg2-binary": ["psycopg2"],
45
+ "mysqlclient": ["MySQLdb"],
46
+ "pyserial": ["serial"],
47
+ "pyopenssl": ["OpenSSL"],
48
+ "pycryptodome": ["Crypto"],
49
+ "pycryptodomex": ["Cryptodome"],
50
+ "setuptools": ["setuptools", "pkg_resources"],
51
+ "python-multipart": ["multipart", "python_multipart"],
52
+ "djangorestframework": ["rest_framework"],
53
+ "django-cors-headers": ["corsheaders"],
54
+ "python-jose": ["jose"],
55
+ "ruamel.yaml": ["ruamel.yaml"],
56
+ "grpcio": ["grpc"],
57
+ "google-api-python-client": ["googleapiclient"],
58
+ "pymupdf": ["fitz", "pymupdf"],
59
+ "python-docx": ["docx"],
60
+ "python-pptx": ["pptx"],
61
+ "websocket-client": ["websocket"],
62
+ "pyzmq": ["zmq"],
63
+ "faiss-cpu": ["faiss"],
64
+ "faiss-gpu": ["faiss"],
65
+ "discord.py": ["discord"],
66
+ "py-cpuinfo": ["cpuinfo"],
67
+ "pygobject": ["gi"],
68
+ "pycairo": ["cairo"],
69
+ "pyqt5": ["PyQt5"],
70
+ "pyqt6": ["PyQt6"],
71
+ "pyside6": ["PySide6"],
72
+ "python-ldap": ["ldap"],
73
+ "pyusb": ["usb"],
74
+ "msgpack-python": ["msgpack"],
75
+ "markdown-it-py": ["markdown_it"],
76
+ "typing-extensions": ["typing_extensions"],
77
+ "google-cloud-storage": ["google.cloud.storage"],
78
+ "google-cloud-bigquery": ["google.cloud.bigquery"],
79
+ "google-cloud-pubsub": ["google.cloud.pubsub", "google.cloud.pubsub_v1"],
80
+ "apache-airflow": ["airflow"],
81
+ "pywin32": ["win32api", "win32con", "win32com", "pythoncom", "pywintypes"],
82
+ };
83
+ const VENV_DIRS = [".venv", "venv", "env", ".env", "virtualenv"];
84
+ /** site-packages directories of virtualenvs at the project root (POSIX and Windows layouts). */
85
+ function sitePackagesDirs(root) {
86
+ const out = [];
87
+ for (const v of VENV_DIRS) {
88
+ const venv = path.join(root, v);
89
+ const win = path.join(venv, "Lib", "site-packages");
90
+ if (fs.existsSync(win))
91
+ out.push(win);
92
+ const lib = path.join(venv, "lib");
93
+ let entries = [];
94
+ try {
95
+ entries = fs.readdirSync(lib);
96
+ }
97
+ catch {
98
+ continue;
99
+ }
100
+ for (const py of entries) {
101
+ const sp = path.join(lib, py, "site-packages");
102
+ if (/^python\d/.test(py) && fs.existsSync(sp))
103
+ out.push(sp);
104
+ }
105
+ }
106
+ return out;
107
+ }
108
+ /** Top-level importable names a RECORD lists: `pkg/__init__.py` -> pkg, `mod.py` -> mod. */
109
+ function namesFromRecord(record) {
110
+ const names = new Set();
111
+ for (const line of record.split(/\r?\n/)) {
112
+ const file = line.split(",")[0]?.trim();
113
+ if (!file || file.startsWith("..") || file.startsWith("/"))
114
+ continue;
115
+ const first = file.split("/")[0];
116
+ if (/\.(?:dist-info|egg-info|data)$/.test(first) || first === "__pycache__" || /\.pth$/.test(first))
117
+ continue;
118
+ if (file.includes("/")) {
119
+ if (/^[A-Za-z_]\w*$/.test(first))
120
+ names.add(first);
121
+ }
122
+ else {
123
+ const mod = /^([A-Za-z_]\w*)(?:\.[\w-]+)?\.(?:py|pyd|so)$/.exec(first)?.[1];
124
+ if (mod)
125
+ names.add(mod);
126
+ }
127
+ }
128
+ return [...names];
129
+ }
130
+ function readInstalledNames(root, dist) {
131
+ const want = normalizePypiName(dist);
132
+ for (const sp of sitePackagesDirs(root)) {
133
+ let entries;
134
+ try {
135
+ entries = fs.readdirSync(sp);
136
+ }
137
+ catch {
138
+ continue;
139
+ }
140
+ for (const entry of entries) {
141
+ const m = /^(.+?)-[^-]+\.(?:dist-info|egg-info)$/.exec(entry);
142
+ if (!m || normalizePypiName(m[1]) !== want)
143
+ continue;
144
+ const meta = path.join(sp, entry);
145
+ const rel = path.relative(root, meta).split(path.sep).join("/");
146
+ for (const [file, read] of [
147
+ ["top_level.txt", (t) => t.split(/\r?\n/).map((l) => l.trim()).filter((l) => /^[A-Za-z_][\w.]*$/.test(l))],
148
+ ["RECORD", namesFromRecord],
149
+ ]) {
150
+ try {
151
+ const names = read(fs.readFileSync(path.join(meta, file), "utf8"));
152
+ if (names.length > 0)
153
+ return { names: [...new Set(names)].sort(), source: "installed-metadata", evidence: `${rel}/${file}` };
154
+ }
155
+ catch {
156
+ // no such file: try the next
157
+ }
158
+ }
159
+ }
160
+ }
161
+ return null;
162
+ }
163
+ /** The module name(s) a distribution is imported as, and how that was established. */
164
+ export function pythonImportNames(root, dist) {
165
+ const installed = root ? readInstalledNames(root, dist) : null;
166
+ if (installed)
167
+ return installed;
168
+ const known = KNOWN_IMPORT_NAMES[normalizePypiName(dist)];
169
+ if (known)
170
+ return { names: known, source: "known-mapping" };
171
+ return { names: [normalizePypiName(dist).replace(/-/g, "_")], source: "assumed" };
172
+ }
173
+ /** True when module `mod` is `name` or inside it (`yaml.loader` for `yaml`). */
174
+ const underModule = (mod, name) => mod === name || mod.startsWith(`${name}.`);
175
+ /**
176
+ * Count imports of any of `names` in one Python file and collect the names it
177
+ * reaches: `from x import A, B` gives A and B; `import x` / `import x as y`
178
+ * gives the attributes used as `x.attr` / `y.attr`.
179
+ */
180
+ export function scanPythonSource(source, names, symbols) {
181
+ let matches = 0;
182
+ const text = source.replace(/\\\r?\n/g, " ");
183
+ const bindings = [];
184
+ // `from x import (a, b,\n c)` and `from x import a, b as c`
185
+ const fromRe = /^[ \t]*from[ \t]+([\w.]+)[ \t]+import[ \t]+(\([^)]*\)|[^\n#]+)/gm;
186
+ for (const m of text.matchAll(fromRe)) {
187
+ if (!names.some((n) => underModule(m[1], n)))
188
+ continue;
189
+ matches++;
190
+ for (const part of m[2].replace(/[()]/g, "").split(",")) {
191
+ const name = part.trim().split(/\s+as\s+/)[0].trim();
192
+ if (/^[A-Za-z_]\w*$/.test(name))
193
+ symbols.add(name);
194
+ }
195
+ }
196
+ const importRe = /^[ \t]*import[ \t]+([^\n#]+)/gm;
197
+ for (const m of text.matchAll(importRe)) {
198
+ for (const part of m[1].split(",")) {
199
+ const [mod, alias] = part.trim().split(/\s+as\s+/).map((s) => s.trim());
200
+ if (!mod || !names.some((n) => underModule(mod, n)))
201
+ continue;
202
+ matches++;
203
+ bindings.push(alias ?? mod);
204
+ }
205
+ }
206
+ // Attribute uses outside the import statements themselves (`from yaml.loader import` names a module, not an attribute).
207
+ const body = text.replace(fromRe, "").replace(importRe, "");
208
+ for (const binding of bindings) {
209
+ const re = new RegExp(String.raw `(?<![\w.])${binding.replace(/\./g, "\\.")}\.([A-Za-z_]\w*)`, "g");
210
+ for (const m of body.matchAll(re))
211
+ symbols.add(m[1]);
212
+ }
213
+ return matches;
214
+ }
215
+ // ---------------------------------------------------------------- Go
216
+ /** True when import path `imp` is module `mod` or a package inside it, not a different major (`mod/v2`). */
217
+ export function goImportInModule(imp, mod) {
218
+ if (imp === mod)
219
+ return true;
220
+ if (!imp.startsWith(`${mod}/`))
221
+ return false;
222
+ const next = imp.slice(mod.length + 1).split("/")[0];
223
+ return !/^v\d+$/.test(next) || goPathMajor(mod) !== null;
224
+ }
225
+ /** The package name an import path is used through: its last element, skipping a `/vN` suffix and gopkg.in's `.vN`. */
226
+ export function goPackageName(imp) {
227
+ const parts = goBasePath(imp).split("/");
228
+ const last = parts[parts.length - 1] ?? imp;
229
+ return last.replace(/\.v\d+$/, "").replace(/^go-/, "").replace(/[-.]/g, "_");
230
+ }
231
+ /**
232
+ * Count imports of a Go module (or packages inside it) in one file and collect
233
+ * the exported identifiers reached through each import's package name.
234
+ */
235
+ export function scanGoSource(source, mod, symbols) {
236
+ const specs = [];
237
+ const spec = /^\s*(?:([A-Za-z_]\w*|\.|_)\s+)?"([^"]+)"/;
238
+ for (const m of source.matchAll(/^\s*import\s*\(([\s\S]*?)\)/gm)) {
239
+ for (const line of m[1].split("\n")) {
240
+ const s = spec.exec(line.replace(/\/\/.*$/, ""));
241
+ if (s)
242
+ specs.push({ alias: s[1] ?? null, path: s[2] });
243
+ }
244
+ }
245
+ for (const m of source.matchAll(/^\s*import\s+(?:([A-Za-z_]\w*|\.|_)\s+)?"([^"]+)"/gm)) {
246
+ specs.push({ alias: m[1] ?? null, path: m[2] });
247
+ }
248
+ let matches = 0;
249
+ for (const s of specs) {
250
+ if (!goImportInModule(s.path, mod))
251
+ continue;
252
+ matches++;
253
+ const local = s.alias ?? goPackageName(s.path);
254
+ if (local === "_" || local === ".")
255
+ continue;
256
+ for (const m of source.matchAll(new RegExp(String.raw `(?<![\w.])${local}\.([A-Z]\w*)`, "g")))
257
+ symbols.add(m[1]);
258
+ }
259
+ return matches;
260
+ }