@preventive/triage 1.0.0-alpha.42 → 1.0.0-alpha.43

package/server-managed/scan-downloads.ts

@@ -0,0 +1,87 @@
+// A scan's packages' downloads, as risk asks them, readied before the
+// packages are assessed (repository-risk.ts), so that no assessment waits on
+// npm's downloads API, which answers some 40 lookups a minute: a package is
+// given to the assessors once its downloads are kept (npm-risk.ts
+// npmReadyScanDownloads), those that can be readied at once (kept already,
+// and unscoped ones, asked in bulk) first, each lockfile's as it is read, and
+// the rest (scoped ones, which npm answers only one by one) one at a time,
+// at the pace npm allows, while the assessors go on with the others.
+import { setTimeout as delay } from 'node:timers/promises'
+import { type Queue, queue } from './queue.ts'
+
+export interface DownloadsSource {
+  // Readies those of `names` that can be at once, `ready` told each.
+  atOnce(names: string[], ready: (name: string) => void, signal: AbortSignal): Promise<void>
+  // Readies one on its own.
+  one(name: string, signal: AbortSignal): Promise<void>
+}
+
+// How npm asking to be asked later is told (`busy`), and waited on: asked
+// again after a wait, up to `retries` times running.
+export interface DownloadsPacing { busy(error: unknown): boolean; retries: number; waitMs: number }
+
+export interface ReadyingDownloads<T> {
+  // Packages to give once their downloads are ready.
+  add(items: T[]): void
+  // Once all added are given, or the scan stops.
+  close(): Promise<void>
+}
+
+// Readies the downloads of the packages added, each given (`give`) once its
+// name's are, or once readying them fails otherwise than npm asking to wait,
+// or npm keeps asking past `retries`: its assessment then asks its own.
+export function readyingDownloads<T extends { name: string }>(source: DownloadsSource, signal: AbortSignal, give: (item: T) => void,
+  { busy, retries, waitMs }: DownloadsPacing): ReadyingDownloads<T> {
+  const waiting = new Map<string, T[]>()
+  const atOnce: Queue<string[]> = queue(signal), oneByOne: Queue<string> = queue(signal)
+  const release = (name: string) => { for (const item of waiting.get(name) ?? []) give(item); waiting.delete(name) }
+  const wait = async (attempt: number) => { try { await delay(waitMs * Math.min(attempt + 1, 5), undefined, { signal }); return true } catch { return false } }
+
+  // A lockfile's names, those that can be readied at once, again after a wait
+  // where npm asks: false once the scan stops.
+  const readyNow = async (names: string[]) => {
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      const left = names.filter(name => waiting.has(name))
+      try { await source.atOnce(left, release, signal); return true } catch (error) {
+        if (signal.aborted) return false
+        if (!busy(error)) { for (const name of left) release(name); return true }
+        if (!await wait(attempt)) return false
+      }
+    }
+    return true
+  }
+  // Each lockfile's, the rest left to oneByOne.
+  const readyAtOnce = async () => {
+    for (let names = await atOnce.take(); names !== undefined; names = await atOnce.take()) {
+      if (!await readyNow(names)) return
+      for (const name of names) if (waiting.has(name)) oneByOne.add(name)
+    }
+  }
+  const readyOneByOne = async () => {
+    for (let name = await oneByOne.take(); name !== undefined; name = await oneByOne.take()) {
+      for (let attempt = 0; attempt <= retries; attempt++) {
+        try { await source.one(name, signal); break } catch (error) {
+          if (signal.aborted) return
+          if (!busy(error) || !await wait(attempt)) break
+        }
+      }
+      release(name)
+    }
+  }
+  const lanes = [readyAtOnce().finally(() => { oneByOne.close() }), readyOneByOne()]
+  return {
+    add(items) {
+      const names: string[] = []
+      for (const item of items) {
+        const kept = waiting.get(item.name)
+        if (kept) kept.push(item)
+        else { waiting.set(item.name, [item]); names.push(item.name) }
+      }
+      if (names.length > 0) atOnce.add(names)
+    },
+    async close() {
+      atOnce.close()
+      await Promise.all(lanes)
+    },
+  }
+}

package/server-managed/sql.ts

@@ -22,7 +22,7 @@ export interface ManagedSqlDriver extends ManagedSql {
 const SINGLE_STATEMENTS = new Set(['getReport', 'getBundle', 'getStorageRow', 'getStorageEncryption',
   'listReports', 'listBundles', 'listReadableBundleIds', 'listAllRepos', 'listSelectedRepos', 'listRepoScopesForUser', 'listTriage',
   'getFeedState', 'getWorkspaceShareFeedState', 'claimMaintenanceLease', 'finishMaintenanceLease', 'cacheRepoDefaultBranch', 'releaseBundleBuildLease',
-  'getUpstreamCacheEntry', 'getUpstreamCacheEntries', 'setUpstreamCacheEntry', 'ensureSessionUploadKey',
+  'getUpstreamCacheEntry', 'getUpstreamCacheEntries', 'setUpstreamCacheEntry', 'setUpstreamCacheEntries', 'ensureSessionUploadKey',
   'listGithubCommits', 'listGithubCommitTags', 'setGithubCommits'])
 // Session presence and upload keys change only their own rows.
 const UNLOCKED_WRITES = new Set(['sessionWithUser', 'viewSessionWithUser', 'ensureSessionUploadKey'])

package/server-managed/team-reports.ts

@@ -5,6 +5,7 @@ import { Buffer } from 'node:buffer'
 import { backfillFindingIds, reportEntries, stampSecurityGroups } from '@preventive/report'
 import { managedFindingSourcePaths, readManagedReport } from '../common/managed/report-content.ts'
 import { filterReportData, projectFinding } from '../common/managed/report-filter.ts'
+import { revalidationLinks, securityLinks } from '../common/security-links.js'
 import type { BlobStore } from './blob-store.ts'
 import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
 import { triageWireEntry } from './triage-response.ts'
@@ -104,18 +105,12 @@ async function buildTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: Tea
   }))
   for (const result of results) if (result.status === 'rejected') throw result.reason
   const globalLinks = snapshot.linkRevision ? await db.getEnabledLinkGroups(snapshot.linkRevision) : []
-  const edges = new Map<string, Set<string>>()
-  const connect = (a: string, b: string) => {
-    if (!edges.has(a)) edges.set(a, new Set())
-    edges.get(a)!.add(b)
-  }
-  // A star is enough for transitive security propagation, without a quadratic
-  // expansion for large links. Unknown IDs can connect known findings.
-  for (const ids of globalLinks.concat(reports.flatMap(report => linksOf(report.data) ?? []))) {
-    for (const id of ids.slice(1)) { connect(ids[0]!, id); connect(id, ids[0]!) }
-  }
   const groups = reports.flatMap(report => groupsOf(report.data).map(group => group.map(f => projectFinding(f, report.data as Finding)!)))
-  stampSecurityGroups(groups, { linkedIds: id => edges.get(id) ?? [] })
+  // Unknown IDs can connect known findings. App findings link to the inputs
+  // they revalidated in other reports, as the App view counts them.
+  stampSecurityGroups(groups, { linkedIds: securityLinks([
+    ...globalLinks, ...reports.flatMap(report => linksOf(report.data) ?? []), ...revalidationLinks(groups.flat()),
+  ]).linkedIds })
   const securityIds = new Set(groups.flat().filter(f => f['isSecurity'] === true).map(f => String(f['id'])))
   const allIds = new Set<string>(), visible: Visibility = new Map()
   for (let i = 0; i < reports.length; i++) {

package/server-managed/upstream-cache.ts

@@ -27,6 +27,9 @@ export interface UpstreamCacheStore {
   // Key → value for those of `keys` that are kept.
   getUpstreamCacheEntries(keys: string[]): Promise<Map<string, string>>
   setUpstreamCacheEntry(key: string, value: string, cachedAt: number): Promise<void>
+  // Each of `entries` (key, value), as setUpstreamCacheEntry keeps one, in one
+  // statement: the last of a key given twice.
+  setUpstreamCacheEntries(entries: [key: string, value: string][], cachedAt: number): Promise<void>
   // Removes those under `prefix` (no LIKE wildcard in it) kept before `before`;
   // how many.
   deleteUpstreamCacheEntries(prefix: string, before: number): Promise<number>
@@ -40,6 +43,8 @@ export function upstreamCacheMethods(db: ManagedSql): UpstreamCacheStore {
     ON CONFLICT (cache_key) DO UPDATE SET value = excluded.value, cached_at = excluded.cached_at
     WHERE managed_upstream_cache.cached_at <= excluded.cached_at`)
   const remove = db.prepare('DELETE FROM managed_upstream_cache WHERE cache_key LIKE ? AND cached_at < ?')
+  // setUpstreamCacheEntries's, by how many it keeps at once.
+  const setMany = new Map<number, ReturnType<ManagedSql['prepare']>>()
   return {
     async getUpstreamCacheEntry(key) {
       const row = await get.get(key) as { value: string } | undefined
@@ -53,6 +58,17 @@ export function upstreamCacheMethods(db: ManagedSql): UpstreamCacheStore {
     async setUpstreamCacheEntry(key, value, cachedAt) {
       await set.run(key, value, cachedAt)
     },
+    async setUpstreamCacheEntries(entries, cachedAt) {
+      const rows = [...new Map(entries)]
+      if (rows.length === 0) return
+      let many = setMany.get(rows.length)
+      if (!many) {
+        setMany.set(rows.length, many = db.prepare(`INSERT INTO managed_upstream_cache (cache_key, value, cached_at) VALUES ${rows.map(() => '(?, ?, ?)').join(', ')}
+          ON CONFLICT (cache_key) DO UPDATE SET value = excluded.value, cached_at = excluded.cached_at
+          WHERE managed_upstream_cache.cached_at <= excluded.cached_at`))
+      }
+      await many.run(...rows.flatMap(([key, value]) => [key, value, cachedAt]))
+    },
     async deleteUpstreamCacheEntries(prefix, before) {
       return Number((await remove.run(`${prefix}%`, before)).changes)
     },