@preventive/upstream 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/advisories.d.ts CHANGED
@@ -50,23 +50,43 @@ export interface AdvisoryOptions {
50
50
  // and its rows take OSV's record of their GHSA, one more request each
51
51
  // (no GitHub client needed).
52
52
  details?: boolean
53
+ // Also asks OSV of each npm package's versions, as cargo and composer
54
+ // packages always are, for what the registry does not have: above all
55
+ // the malicious packages' records (MAL-...), which it has none of, as
56
+ // those of npm releases taken down that GitHub's database has no
57
+ // advisory of. A version a registry row reports under the same advisory,
58
+ // its id among an OSV record's id and aliases, is left to that row,
59
+ // which takes the record's ids among its aliases; the versions a record
60
+ // of the row's own id covers beyond the registry's range join it too.
61
+ // Sends each npm package's name and versions to OSV, so not for a
62
+ // private package.
63
+ osv?: boolean
53
64
  }
54
65
 
55
66
  // One advisory on one package, with the versions asked about that it
56
- // covers (never empty).
67
+ // covers (never empty): one row an id, however many of its sources answer
68
+ // it, or however often one does.
57
69
  export interface Advisory {
58
70
  ecosystem: Ecosystem
59
71
  name: string
60
- // `registry`: npm's, as `npm audit` has it, one row per vulnerable
61
- // range. `osv`: OSV's records for cargo and composer, RustSec's, GitHub's
62
- // and malicious packages' (MAL-...) among them; of the records naming each
63
- // other as aliases, directly or through others, a version is reported
64
- // under one only: RustSec's, else the GHSA, else the first by id. Each
65
- // has the others' ids among its aliases.
66
- // `repository`: published on the package's repository, one row per
67
- // range, holding only the versions the others' answer does not report
68
- // under that GHSA. The only source for `soldeer` and `github`, where
69
- // every range counts, whichever package it names.
72
+ // `registry`: npm's, as `npm audit` has it. `osv`: OSV's records for
73
+ // cargo and composer, and with `osv` for npm, the versions they cover
74
+ // that no registry row reports under the same advisory; RustSec's,
75
+ // GitHub's and malicious packages' (MAL-...) among them; of the records
76
+ // naming each other as aliases, directly or through others, a version is
77
+ // reported under one only: RustSec's, else the GHSA, else the first by
78
+ // id. Each has the others' ids among its aliases, and stays a row of its
79
+ // own, as its source rates it.
80
+ // `repository`: published on the package's repository, holding only the
81
+ // versions the others' answer does not report under that GHSA. The only
82
+ // source for `soldeer` and `github`, where every range counts, whichever
83
+ // package it names.
84
+ // An id answered more than once -- by npm's registry once a range, by
85
+ // OSV's record of it or its repository for versions beyond the registry's
86
+ // range, by its repository once a range -- is one row: the registry's,
87
+ // else OSV's, else the repository's, with what it lacks of theirs (title,
88
+ // details), the worst severity of them, the highest CVSS score with its
89
+ // own vector, and all their ranges, CWEs and versions.
70
90
  source: 'registry' | 'osv' | 'repository'
71
91
  // A GHSA, RUSTSEC-..., MAL-..., DRUPAL-CORE-... or another OSV id, or
72
92
  // npm:<id> for a registry row without a GHSA.
@@ -81,6 +101,8 @@ export interface Advisory {
81
101
  // (https://osv.dev/vulnerability/RUSTSEC-...). Built, never fetched: only
82
102
  // a registry row without a GHSA (npm:<id>) has none.
83
103
  url?: string
104
+ // The advisory's other ids: an `osv` row's record's own and its group's,
105
+ // and with `osv`, a registry row's OSV records' of it.
84
106
  aliases: string[]
85
107
  title?: string
86
108
  // With `details`, the advisory's full text, Markdown as published: an
@@ -96,11 +118,17 @@ export interface Advisory {
96
118
  cvssVector?: string
97
119
  cwe: string[]
98
120
  // As written: npm's syntax from the registry, GitHub's
99
- // (`>= 1.0.0, < 1.2.6`) from a repository; OSV rows have none. A
121
+ // (`>= 1.0.0, < 1.2.6`) from a repository; OSV's records have none. A
100
122
  // repository range semver cannot read, or none, covers every version.
123
+ // Each range its id was answered with that covers an asked version,
124
+ // joined by ` || `; none ('') where one is none. A version only OSV
125
+ // reports may be outside it: hermes-engine's GHSA-f5x2-xv93-4p23 is
126
+ // `<=0.4.3` on npm, the last version known affected, and fixed in 0.5.2
127
+ // by OSV's record of it, which reports 0.5.0 too.
101
128
  range?: string
102
129
  // RustSec's kind for an advisory that is not a vulnerability as such:
103
- // `unmaintained`, `unsound`, `notice`.
130
+ // `unmaintained`, `unsound`, `notice`. For an id answered more than once,
131
+ // only where each answer gives it.
104
132
  informational?: string
105
133
  versions: string[]
106
134
  }
@@ -110,14 +138,18 @@ export interface Advisory {
110
138
  // merged, and must not name two repositories. npm is asked 250 names a
111
139
  // request, OSV 1000 versions, Soldeer one project, and with `details`, OSV
112
140
  // one GHSA among npm's rows; a project Soldeer does not have or that names
113
- // no GitHub repository, and a repository gone, renamed or blocked, add
114
- // nothing, and a GHSA OSV does not have adds no `details`. Any other
115
- // failure, a malformed answer, or one about something not asked, throws:
116
- // nothing is left out quietly.
141
+ // no GitHub repository, a repository gone or blocked, and one GitHub
142
+ // redirects anywhere but to a repository by its id, add nothing, and a
143
+ // GHSA OSV does not have adds no `details`. A repository renamed or
144
+ // transferred, which GitHub redirects to it by its id, is listed by the
145
+ // name it has now, as getRepo's `followRedirect` (github.js) finds it, its
146
+ // rows' `url` on it. Any other failure, a malformed answer, or one about
147
+ // something not asked, throws: nothing is left out quietly.
117
148
  // A repository's listing is kept for 90 minutes, as one entry for every
118
- // spelling of its name, holding only what rows are made from, `details`
119
- // included: in setCacheDir's cache gzipped, in a `cache` store as it is,
120
- // or with `cache` false, nowhere. 90 minutes is how late an advisory its
149
+ // spelling of the name it is asked by, one renamed or transferred too,
150
+ // holding only what rows are made from, `details` included: in
151
+ // setCacheDir's cache gzipped, in a `cache` store as it is, or with
152
+ // `cache` false, nowhere. 90 minutes is how late an advisory its
121
153
  // maintainer publishes can be seen. One gone or malformed is not kept.
122
154
  // Sorted by ecosystem and name. Versions are matched by npm's semver, from
123
155
  // the npm beside node, or by the semver peer where there is no npm.
package/github.d.ts CHANGED
@@ -7,7 +7,7 @@
7
7
  // encoded segments and has to come back out of URL parsing unchanged.
8
8
  // Redirects are refused, so an answer is about the repo asked for, except
9
9
  // the tarballs' and, where asked, getRepo's for a repo renamed or
10
- // transferred. A response is read up to a size limit (64 MiB of JSON, 128
10
+ // transferred. A response is read up to a size limit (128 MiB of JSON, 128
11
11
  // MiB of a file, 512 MiB of a tarball) and within a timeout (30 seconds, 5
12
12
  // minutes for a tarball), and what it says goes into an error message
13
13
  // escaped.
package/npm.d.ts CHANGED
@@ -127,7 +127,11 @@ export interface Dist {
127
127
  // `bugs` an object, of which only the strings named below are read. Each is
128
128
  // absent where the document has none, or has another shape, or an object
129
129
  // with none of those strings. All of it is the publisher's word, held to no
130
- // repository.
130
+ // repository. `unpackedSize` is the document's dist.unpackedSize: the bytes
131
+ // of the files in the tarball, as the registry counted them when the
132
+ // version was published, held to no tarball. It is absent where the
133
+ // document has none, as for a version published before the registry
134
+ // counted them, or has one that is not a whole number of bytes.
131
135
  export interface PackageMeta {
132
136
  name: string
133
137
  version: string
@@ -136,6 +140,7 @@ export interface PackageMeta {
136
140
  repository?: string | PackageRepository
137
141
  homepage?: string
138
142
  bugs?: string | PackageBugs
143
+ unpackedSize?: number
139
144
  }
140
145
 
141
146
  export interface PackageRepository {
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@preventive/upstream",
3
- "version": "1.0.1",
4
- "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
3
+ "version": "1.2.0",
4
+ "description": "Minimal npm, crates.io, Soldeer, GitHub and git checkout clients",
5
5
  "type": "module",
6
6
  "exports": {
7
7
  "./advisories.js": {
@@ -1,9 +1,9 @@
1
1
  import assert from 'node:assert/strict'
2
2
 
3
- import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
3
+ import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isRepo, isStrings } from '../args.js'
4
4
  import { readRecord, writeRecord } from '../cache.js'
5
5
  import { isGone } from '../github/client.js'
6
- import { recover } from '../http.js'
6
+ import { HttpError, recover } from '../http.js'
7
7
  import { pool } from '../pool.js'
8
8
  import { isExactVersion } from '../semver.js'
9
9
  import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
@@ -21,7 +21,7 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
21
21
  && advisory.state === 'published' && isText(advisory.summary)
22
22
  && (advisory.vulnerabilities == null || Array.isArray(advisory.vulnerabilities)) && (advisory.cwe_ids == null || isStrings(advisory.cwe_ids))
23
23
 
24
- export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
24
+ export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function' && typeof value.getRepo === 'function')
25
25
 
26
26
  // A maintainer may give a vulnerable range only its lower bound, and the
27
27
  // version that fixed it apart (`>= 5.0.0-beta.1`, patched in `5.0.0-rc.2`),
@@ -67,18 +67,37 @@ const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhs
67
67
  && (advisory.description === undefined || isText(advisory.description))
68
68
  && Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
69
69
 
70
- // A repository's listing, digested, through the cache: GitHub's names are
71
- // case-insensitive, so one entry answers every spelling. A repository gone,
72
- // renamed or blocked has none, and that is not kept.
70
+ // GitHub answers for a repository renamed or transferred with a redirect,
71
+ // which getRepo follows to the repository it is now, listed by that name.
72
+ const isRedirect = (err) => err instanceof HttpError && err.status >= 300 && err.status < 400
73
+
74
+ async function listingOf(github, repo) {
75
+ try {
76
+ return { repo, list: await github.listRepoAdvisories({ repo }) }
77
+ } catch (err) {
78
+ if (!isRedirect(err)) throw err
79
+ }
80
+ const moved = (await github.getRepo({ repo, followRedirect: true })).full_name
81
+ return { repo: moved, list: await github.listRepoAdvisories({ repo: moved }) }
82
+ }
83
+
84
+ // A repository's listing, digested, through the cache, and the repository
85
+ // it is of: GitHub's names are case-insensitive, so one entry answers every
86
+ // spelling, and one renamed or transferred is kept under the name asked,
87
+ // with `repo` the one it has now. A repository gone or blocked, or
88
+ // redirected anywhere getRepo does not follow, has none, and that is not
89
+ // kept.
73
90
  async function listAdvisories(github, repo, cache) {
74
91
  const name = repo.toLowerCase()
75
92
  const entry = await readRecord(DIR, name, cache, LISTING_TTL_MS)
76
- if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
77
- const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
78
- if (list === null) return []
79
- const advisories = digest(repo, list)
80
- if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, cache)
81
- return advisories
93
+ if (entry?.v === VERSION && (entry.repo === undefined || isRepo(entry.repo)) && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) {
94
+ return { repo: entry.repo ?? repo, advisories: entry.advisories }
95
+ }
96
+ const found = await listingOf(github, repo).catch(recover((err) => isGone(err) || isRedirect(err), null))
97
+ if (found === null) return { repo, advisories: [] }
98
+ const advisories = digest(found.repo, found.list)
99
+ if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, ...(found.repo !== repo && { repo: found.repo }), advisories }, cache)
100
+ return { repo: found.repo, advisories }
82
101
  }
83
102
 
84
103
  // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
@@ -109,10 +128,10 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
109
128
  async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
110
129
  // GitHub's names are case-insensitive: one spelling asks for all.
111
130
  const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
112
- const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
131
+ const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ names, ...(await listAdvisories(github, repoOf(names[0]), cache)) }))
113
132
  const rows = new Map()
114
- for (const { repo, names, list } of listed) {
115
- for (const advisory of list) {
133
+ for (const { repo, names, advisories } of listed) {
134
+ for (const advisory of advisories) {
116
135
  for (const { range, ...pkg } of advisory.ranges) {
117
136
  for (const name of names.filter((candidate) => takes(candidate, pkg))) {
118
137
  rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
@@ -34,8 +34,59 @@ function collect(packages) {
34
34
  return byEcosystem
35
35
  }
36
36
 
37
+ // Which of an advisory's rows stands for it: npm's registry's, then OSV's,
38
+ // then its repository's.
39
+ const SOURCES = ['registry', 'osv', 'repository']
40
+ const SEVERITIES = ['critical', 'high', 'moderate', 'low', 'info']
41
+ const worse = (a, b) => (b === undefined || (a !== undefined && SEVERITIES.indexOf(a) < SEVERITIES.indexOf(b)) ? a : b)
42
+
43
+ // An advisory's rows as one: the first's by source, what it lacks from the
44
+ // first of the rest that has it, the others' aliases, and of them all the
45
+ // worst severity, the highest CVSS score with its own vector (else the first
46
+ // vector), the ranges (` || `), CWEs and versions (in the order `asked` has
47
+ // them). RustSec's kind of advisory that is not a vulnerability as such only
48
+ // where each of them has that kind: a row without one is a vulnerability.
49
+ function joinRows(rows, asked) {
50
+ const [head, ...rest] = rows.toSorted((a, b) => SOURCES.indexOf(a.source) - SOURCES.indexOf(b.source))
51
+ const all = [head, ...rest]
52
+ const ranges = [...new Set(all.map((row) => row.range).filter((range) => range !== undefined))]
53
+ const versions = new Set(all.flatMap((row) => row.versions))
54
+ const joined = {
55
+ ...head,
56
+ aliases: [...new Set(all.flatMap((row) => row.aliases))].filter((id) => id !== head.id),
57
+ cwe: [...new Set(all.flatMap((row) => row.cwe))],
58
+ versions: asked.filter((version) => versions.has(version)),
59
+ }
60
+ if (rest.some((row) => row.informational !== head.informational)) delete joined.informational
61
+ for (const key of ['title', 'details']) {
62
+ const from = joined[key] === undefined && rest.find((row) => row[key] !== undefined)
63
+ if (from) joined[key] = from[key]
64
+ }
65
+ const severity = all.map((row) => row.severity).reduce(worse)
66
+ if (severity !== undefined) joined.severity = severity
67
+ const scored = all.filter((row) => row.cvss !== undefined).toSorted((a, b) => b.cvss - a.cvss)[0] ?? all.find((row) => row.cvssVector !== undefined)
68
+ delete joined.cvss
69
+ delete joined.cvssVector
70
+ if (scored?.cvss !== undefined) joined.cvss = scored.cvss
71
+ if (scored?.cvssVector !== undefined) joined.cvssVector = scored.cvssVector
72
+ // A range of none ('') covers every version, as they then do together.
73
+ if (ranges.length > 0) joined.range = ranges.includes('') ? '' : ranges.join(' || ')
74
+ return joined
75
+ }
76
+
77
+ // One row for each advisory id on a package, where its sources answer more
78
+ // than one: npm's registry one per range, and OSV's record of it, or its
79
+ // repository, another for the versions they cover beyond that. Records of
80
+ // other ids that name it as an alias stay rows of their own, each version
81
+ // reported under one, as their sources rate them. Rows keep the place of the
82
+ // first answered.
83
+ function oneRowPerAdvisory(rows, asked) {
84
+ const byId = Map.groupBy(rows, (row) => `${row.name}\0${row.id}`)
85
+ return [...byId.values()].map((same) => (same.length === 1 ? same[0] : joinRows(same, asked.get(same[0].name))))
86
+ }
87
+
37
88
  export async function advisories(packages, options = {}) {
38
- assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCache) })
89
+ assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), osv: optional(assertBoolean), cache: optional(assertCache) })
39
90
  assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
40
91
  const byEcosystem = collect(packages)
41
92
  for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
@@ -44,8 +95,8 @@ export async function advisories(packages, options = {}) {
44
95
  const names = [...named.keys()].toSorted()
45
96
  const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
46
97
  const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
47
- const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
48
- return rows.map((row) => ({ ecosystem, ...row }))
98
+ const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, osv: options.osv === true, known, cache: options.cache })
99
+ return oneRowPerAdvisory(rows.filter((row) => row.versions.length > 0), asked).map((row) => ({ ecosystem, ...row }))
49
100
  }))
50
- return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
101
+ return found.flat().toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
51
102
  }
@@ -6,7 +6,7 @@ import { chunks, pool } from '../pool.js'
6
6
  import { compareVersions } from '../semver.js'
7
7
  import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
8
8
  import { withRepositories } from './github.js'
9
- import { RECORDS_AT_ONCE, getVuln } from './osv.js'
9
+ import { RECORDS_AT_ONCE, getVuln, osvRows } from './osv.js'
10
10
  import { npmRepos } from './repos.js'
11
11
 
12
12
  const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
@@ -52,21 +52,48 @@ async function registryAdvisories(asked) {
52
52
 
53
53
  // The registry's rows carry no text: each GHSA's is OSV's record of it,
54
54
  // GitHub's advisory database as published, asked once for the rows that
55
- // cover an asked version. One OSV does not have yet has none.
55
+ // cover an asked version, but where `osv` already read it. One OSV does
56
+ // not have yet has none.
56
57
  async function withDetails(rows) {
57
- const ghsas = [...new Set(rows.filter((row) => row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
58
+ const asks = (row) => row.source === 'registry' && row.details === undefined
59
+ const ghsas = [...new Set(rows.filter((row) => asks(row) && row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
58
60
  const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
59
61
  const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
60
- return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
62
+ return rows.map((row) => (asks(row) ? { ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) } : row))
61
63
  }
62
64
 
63
- // What `npm audit` asks the registry, one row per vulnerable range.
65
+ // With `osv`, OSV's records of the asked versions besides the registry's
66
+ // rows, the malicious packages' (MAL-...), which the registry has none
67
+ // of, among them. A version a registry row reports under the same
68
+ // advisory, the row's id one of an OSV row's ids or aliases, is left to
69
+ // that row, which takes the OSV row's ids among its aliases, and its text
70
+ // where it is the same record; an OSV row keeps the versions no such row
71
+ // reports, which advisories() joins to a registry row of its own id.
72
+ function withOsv(registry, osv) {
73
+ const rows = registry.map((row) => ({ ...row }))
74
+ const added = []
75
+ for (const record of osv) {
76
+ const ids = [record.id, ...record.aliases]
77
+ const same = rows.filter((row) => row.name === record.name && ids.includes(row.id) && row.versions.some((version) => record.versions.includes(version)))
78
+ for (const row of same) {
79
+ row.aliases = [...new Set([...row.aliases, ...ids])].filter((id) => id !== row.id)
80
+ if (record.id === row.id && record.details !== undefined) row.details = record.details
81
+ }
82
+ const versions = record.versions.filter((version) => !same.some((row) => row.versions.includes(version)))
83
+ if (versions.length > 0) added.push({ ...record, versions })
84
+ }
85
+ return [...rows, ...added]
86
+ }
87
+
88
+ // What `npm audit` asks the registry, one row per vulnerable range, and
89
+ // with `osv`, what OSV has besides.
64
90
  export const NPM = {
65
91
  assertName: assertPackageName,
66
92
  assertVersion: assertPackageVersion,
67
93
  compare: compareVersions,
68
94
  advisories: async (asked, options) => {
69
- const rows = await registryAdvisories(asked)
95
+ const [registry, osv] = await Promise.all([registryAdvisories(asked), options.osv ? osvRows({ osv: 'npm' }, asked, options) : []])
96
+ const rows = withOsv(registry, osv)
70
97
  return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
71
98
  },
72
99
  }
@@ -84,9 +84,9 @@ function toAdvisory(ecosystem, name, versions, record, details) {
84
84
  // publishes comes back under each one's id, the records naming each other
85
85
  // as aliases, either way round, directly or through others: one group, in
86
86
  // which each version is reported under the first answered for it, and
87
- // each record has the others' ids as aliases.
88
- async function osvAdvisories(ecosystem, asked, options) {
89
- const { osv, github, lookUp, covers } = ecosystem
87
+ // each record has the others' ids as aliases. `osv` is OSV's name for the
88
+ // ecosystem.
89
+ export async function osvRows({ osv }, asked, { details }) {
90
90
  const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
91
91
  const hits = new Map() // id -> name -> versions, in `list` order
92
92
  for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
@@ -118,9 +118,14 @@ async function osvAdvisories(ecosystem, asked, options) {
118
118
  const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
119
119
  for (const [name, versions] of hits.get(record.id)) {
120
120
  const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
121
- if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, options.details))
121
+ if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, details))
122
122
  }
123
123
  }
124
124
  rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))
125
- return await withRepositories(rows, asked, options, { ecosystem: github, lookUp, covers })
125
+ return rows
126
+ }
127
+
128
+ async function osvAdvisories(ecosystem, asked, options) {
129
+ const { github, lookUp, covers } = ecosystem
130
+ return await withRepositories(await osvRows(ecosystem, asked, options), asked, options, { ecosystem: github, lookUp, covers })
126
131
  }
package/src/http.js CHANGED
@@ -26,7 +26,7 @@ const isQuery = ([key, value]) => isQueryKey(key) && (key.endsWith('[]') ? Array
26
26
  // A line break in a value would start a header of its own.
27
27
  const isHeader = ([name, value]) => /^[A-Za-z][\w-]*$/u.test(name) && matches(/^[ -~]*$/u)(value)
28
28
  const LIMITS = {
29
- json: { bytes: 64 * 1024 * 1024, ms: 30_000 },
29
+ json: { bytes: 128 * 1024 * 1024, ms: 30_000 },
30
30
  text: { bytes: 128 * 1024 * 1024, ms: 30_000 },
31
31
  bytes: { bytes: MAX_BYTES, ms: 300_000 },
32
32
  }
@@ -73,24 +73,28 @@ async function npmCachedVersion(name, version, given) {
73
73
  return undefined
74
74
  }
75
75
 
76
+ const countOr = (value) => (Number.isSafeInteger(value) && value >= 0 ? value : undefined)
77
+
76
78
  // The version's document: with the `dist` the caller has, npm's own cache's
77
79
  // where it matches (npmCachedVersion), and neither `cache` nor the cache
78
80
  // set is read or written; else as getVersion has it. Beside its dist, the
79
81
  // document's gitHead where that is a full commit id, and its repository,
80
- // homepage and bugs, in the shapes a package.json gives them. Each but
81
- // dist is left out where there is none.
82
+ // homepage and bugs, in the shapes a package.json gives them; and its
83
+ // dist's unpackedSize where that is a count. Each but dist is left out
84
+ // where there is none.
82
85
  export async function getMeta(name, version, options = {}) {
83
86
  assertPackage('getMeta', name, version)
84
87
  assertArgs('getMeta', options, { dist: null, cache: optional(assertCache) })
85
88
  const given = options.dist === undefined ? undefined : checkedDist('getMeta', name, version, options.dist)
86
89
  const { dist, json } = (given && await npmCachedVersion(name, version, given)) ?? await getVersion('getMeta', name, version, given, options.cache)
87
- const about = {
90
+ const said = {
88
91
  gitHead: isSha(own(json, 'gitHead')) ? json.gitHead : undefined,
89
92
  repository: linkOf(own(json, 'repository'), ['type', 'url', 'directory']),
90
93
  homepage: stringOr(own(json, 'homepage')),
91
94
  bugs: linkOf(own(json, 'bugs'), ['url', 'email']),
95
+ unpackedSize: countOr(own(json.dist, 'unpackedSize')),
92
96
  }
93
- return { name, version, dist, ...Object.fromEntries(Object.entries(about).filter(([, value]) => value !== undefined)) }
97
+ return { name, version, dist, ...Object.fromEntries(Object.entries(said).filter(([, value]) => value !== undefined)) }
94
98
  }
95
99
 
96
100
  // Only the whole package's document has `time`: megabytes for some.