@preventive/triage 1.0.0-alpha.42 → 1.0.0-alpha.44

package/server-managed/scan-downloads.ts

@@ -0,0 +1,87 @@
+// A scan's packages' downloads, as risk asks them, readied before the
+// packages are assessed (repository-risk.ts), so that no assessment waits on
+// npm's downloads API, which answers some 40 lookups a minute: a package is
+// given to the assessors once its downloads are kept (npm-risk.ts
+// npmReadyScanDownloads), those that can be readied at once (kept already,
+// and unscoped ones, asked in bulk) first, each lockfile's as it is read, and
+// the rest (scoped ones, which npm answers only one by one) one at a time,
+// at the pace npm allows, while the assessors go on with the others.
+import { setTimeout as delay } from 'node:timers/promises'
+import { type Queue, queue } from './queue.ts'
+
+export interface DownloadsSource {
+  // Readies those of `names` that can be at once, `ready` told each.
+  atOnce(names: string[], ready: (name: string) => void, signal: AbortSignal): Promise<void>
+  // Readies one on its own.
+  one(name: string, signal: AbortSignal): Promise<void>
+}
+
+// How npm asking to be asked later is told (`busy`), and waited on: asked
+// again after a wait, up to `retries` times running.
+export interface DownloadsPacing { busy(error: unknown): boolean; retries: number; waitMs: number }
+
+export interface ReadyingDownloads<T> {
+  // Packages to give once their downloads are ready.
+  add(items: T[]): void
+  // Once all added are given, or the scan stops.
+  close(): Promise<void>
+}
+
+// Readies the downloads of the packages added, each given (`give`) once its
+// name's are, or once readying them fails otherwise than npm asking to wait,
+// or npm keeps asking past `retries`: its assessment then asks its own.
+export function readyingDownloads<T extends { name: string }>(source: DownloadsSource, signal: AbortSignal, give: (item: T) => void,
+  { busy, retries, waitMs }: DownloadsPacing): ReadyingDownloads<T> {
+  const waiting = new Map<string, T[]>()
+  const atOnce: Queue<string[]> = queue(signal), oneByOne: Queue<string> = queue(signal)
+  const release = (name: string) => { for (const item of waiting.get(name) ?? []) give(item); waiting.delete(name) }
+  const wait = async (attempt: number) => { try { await delay(waitMs * Math.min(attempt + 1, 5), undefined, { signal }); return true } catch { return false } }
+
+  // A lockfile's names, those that can be readied at once, again after a wait
+  // where npm asks: false once the scan stops.
+  const readyNow = async (names: string[]) => {
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      const left = names.filter(name => waiting.has(name))
+      try { await source.atOnce(left, release, signal); return true } catch (error) {
+        if (signal.aborted) return false
+        if (!busy(error)) { for (const name of left) release(name); return true }
+        if (!await wait(attempt)) return false
+      }
+    }
+    return true
+  }
+  // Each lockfile's, the rest left to oneByOne.
+  const readyAtOnce = async () => {
+    for (let names = await atOnce.take(); names !== undefined; names = await atOnce.take()) {
+      if (!await readyNow(names)) return
+      for (const name of names) if (waiting.has(name)) oneByOne.add(name)
+    }
+  }
+  const readyOneByOne = async () => {
+    for (let name = await oneByOne.take(); name !== undefined; name = await oneByOne.take()) {
+      for (let attempt = 0; attempt <= retries; attempt++) {
+        try { await source.one(name, signal); break } catch (error) {
+          if (signal.aborted) return
+          if (!busy(error) || !await wait(attempt)) break
+        }
+      }
+      release(name)
+    }
+  }
+  const lanes = [readyAtOnce().finally(() => { oneByOne.close() }), readyOneByOne()]
+  return {
+    add(items) {
+      const names: string[] = []
+      for (const item of items) {
+        const kept = waiting.get(item.name)
+        if (kept) kept.push(item)
+        else { waiting.set(item.name, [item]); names.push(item.name) }
+      }
+      if (names.length > 0) atOnce.add(names)
+    },
+    async close() {
+      atOnce.close()
+      await Promise.all(lanes)
+    },
+  }
+}

package/server-managed/sql.ts

@@ -22,7 +22,7 @@ export interface ManagedSqlDriver extends ManagedSql {
 const SINGLE_STATEMENTS = new Set(['getReport', 'getBundle', 'getStorageRow', 'getStorageEncryption',
   'listReports', 'listBundles', 'listReadableBundleIds', 'listAllRepos', 'listSelectedRepos', 'listRepoScopesForUser', 'listTriage',
   'getFeedState', 'getWorkspaceShareFeedState', 'claimMaintenanceLease', 'finishMaintenanceLease', 'cacheRepoDefaultBranch', 'releaseBundleBuildLease',
-  'getUpstreamCacheEntry', 'getUpstreamCacheEntries', 'setUpstreamCacheEntry', 'ensureSessionUploadKey',
+  'getUpstreamCacheEntry', 'getUpstreamCacheEntries', 'setUpstreamCacheEntry', 'setUpstreamCacheEntries', 'ensureSessionUploadKey',
   'listGithubCommits', 'listGithubCommitTags', 'setGithubCommits'])
 // Session presence and upload keys change only their own rows.
 const UNLOCKED_WRITES = new Set(['sessionWithUser', 'viewSessionWithUser', 'ensureSessionUploadKey'])

package/server-managed/team-app.ts

@@ -4,6 +4,11 @@
 // loads. Results are keyed by the catalog's own content keys: an unchanged
 // catalog reuses them, and every change that can move a result also changes
 // the catalog revision that clients refresh on.
+//
+// The same read names the reports a restricted reader would find nothing in:
+// under -deps or -security the catalog leaves those out of the team's list.
+// Readers with both permissions keep every report, empty or not.
+import { reportEntries } from '@preventive/report'
 import { type TeamAppMetadata, managedTeamAppMetadata } from '../common/managed/team-app.js'
 import type { BlobStore } from './blob-store.ts'
 import type { ManagedDb, TeamReportAccessSnapshot, UserTeam } from './db.ts'
@@ -11,12 +16,14 @@ import type { WorkspaceShareSnapshot } from './workspace-shares.ts'
 import { TeamReportsError, loadTeamReportsResponse } from './team-reports.ts'
 
 export type { TeamAppMetadata }
+export interface TeamWorkspaceSummary { app: TeamAppMetadata; empty: readonly string[] }
+const NO_APP: TeamWorkspaceSummary = { app: { appMode: false }, empty: [] }
 
 const CACHE_MAX_ENTRIES = 4096
 // Each workspace build already reads four reports at a time.
 const CLASSIFY_CONCURRENCY = 2
-const caches = new WeakMap<ManagedDb, Map<string, TeamAppMetadata>>()
-const pendingJobs = new WeakMap<ManagedDb, Map<string, Promise<TeamAppMetadata | null>>>()
+const caches = new WeakMap<ManagedDb, Map<string, TeamWorkspaceSummary>>()
+const pendingJobs = new WeakMap<ManagedDb, Map<string, Promise<TeamWorkspaceSummary | null>>>()
 
 // Published reports with their content keys, and the team's own key (access,
 // scopes and links). Equal keys describe the same filtered workspace, whoever
@@ -29,28 +36,36 @@ function published(team: UserTeam) {
   return team.reports.filter(report => report.visible !== false)
 }
 
-function remember(db: ManagedDb, key: string, app: TeamAppMetadata): TeamAppMetadata {
+function remember(db: ManagedDb, key: string, summary: TeamWorkspaceSummary): TeamWorkspaceSummary {
   let cache = caches.get(db)
   if (!cache) { cache = new Map(); caches.set(db, cache) }
   cache.delete(key)
-  cache.set(key, app)
+  cache.set(key, summary)
   for (const oldest of cache.keys()) {
     if (cache.size <= CACHE_MAX_ENTRIES) break
     cache.delete(oldest)
   }
-  return app
+  return summary
+}
+
+// A links document counts by its links, every other report by its findings.
+function hasContent(data: unknown): boolean {
+  const record = data as { source?: unknown; links?: unknown } | null
+  if (record?.source === 'links' && Array.isArray(record.links)) return record.links.length > 0
+  return (reportEntries(data) ?? []).some(entry => !Array.isArray(entry) || entry.length > 0)
 }
 
 // Null when the team cannot be classified right now: its catalog went stale,
 // the access snapshot is gone, or storage failed. The sidebar then keeps the
-// team expanded, and a later catalog asks again. Oversized and unreadable
-// workspaces never have an App view, so they are remembered as such.
+// team expanded and lists every report, and a later catalog asks again.
+// Oversized and unreadable workspaces never have an App view, so they are
+// remembered as such, hiding nothing.
 async function classify(db: ManagedDb, store: BlobStore, key: string, team: UserTeam,
-  snapshot: () => Promise<TeamReportAccessSnapshot | null>): Promise<TeamAppMetadata | null> {
+  snapshot: () => Promise<TeamReportAccessSnapshot | null>): Promise<TeamWorkspaceSummary | null> {
   const cached = caches.get(db)?.get(key)
   if (cached) return remember(db, key, cached)
   const ids = new Set(published(team).map(report => report.id))
-  if (ids.size === 0) return remember(db, key, { appMode: false })
+  if (ids.size === 0) return remember(db, key, NO_APP)
   let jobs = pendingJobs.get(db)
   if (!jobs) { jobs = new Map(); pendingJobs.set(db, jobs) }
   const existing = jobs.get(key)
@@ -60,11 +75,13 @@ async function classify(db: ManagedDb, store: BlobStore, key: string, team: User
     // A catalog read before a concurrent publication names another report set.
     if (!access?.teamId || access.reports.length !== ids.size || access.reports.some(report => !ids.has(report.id))) return null
     const body = JSON.parse((await loadTeamReportsResponse(db, store, access)).toString('utf8')) as {
-      reports: { filename: string; data: unknown }[]; links?: string[][]
+      reports: { id: string; filename: string; data: unknown }[]; links?: string[][]
     }
-    return remember(db, key, managedTeamAppMetadata(Object.assign(body.reports, { links: body.links ?? [] })))
+    const restricted = access.reports.some(report => !report.permissions.dependencies || !report.permissions.security)
+    const empty = restricted ? body.reports.filter(report => !hasContent(report.data)).map(report => report.id) : []
+    return remember(db, key, { app: managedTeamAppMetadata(Object.assign(body.reports, { links: body.links ?? [] })), empty })
   })().catch((error: unknown) => {
-    if (error instanceof TeamReportsError && [413, 422].includes(error.status)) return remember(db, key, { appMode: false })
+    if (error instanceof TeamReportsError && [413, 422].includes(error.status)) return remember(db, key, NO_APP)
     if (!(error instanceof TeamReportsError)) console.warn('managed: team App classification failed:', error)
     return null
   })
@@ -72,10 +89,17 @@ async function classify(db: ManagedDb, store: BlobStore, key: string, team: User
   try { return await job } finally { jobs.delete(key) }
 }
 
+// The catalog's view of a classified team: its App classification, and its
+// reports without the ones its restricted reader would find empty.
+export function withWorkspaceSummary<T extends { reports: readonly { id: string }[] }>(team: T, summary: TeamWorkspaceSummary | null | undefined) {
+  const empty = new Set(summary?.empty)
+  return { ...team, app: summary?.app ?? null, reports: team.reports.filter(report => !empty.has(report.id)) }
+}
+
 // Classify a user's catalog, keyed by teamAppKey. `known` carries results from
 // an earlier read of the same request, so only changed teams are classified.
 export async function teamAppStates(db: ManagedDb, store: BlobStore, sessionId: string, teams: readonly UserTeam[],
-  known: ReadonlyMap<string, TeamAppMetadata | null> = new Map()): Promise<Map<string, TeamAppMetadata | null>> {
+  known: ReadonlyMap<string, TeamWorkspaceSummary | null> = new Map()): Promise<Map<string, TeamWorkspaceSummary | null>> {
   const states = new Map(known)
   const missing = [...new Map(teams.map(team => [teamAppKey(team), team])).entries()].filter(([key]) => !states.has(key))
   let next = 0
@@ -89,6 +113,6 @@ export async function teamAppStates(db: ManagedDb, store: BlobStore, sessionId:
 }
 
 // A public link's workspace is its own access snapshot.
-export function sharedTeamApp(db: ManagedDb, store: BlobStore, snapshot: WorkspaceShareSnapshot): Promise<TeamAppMetadata | null> {
+export function sharedTeamApp(db: ManagedDb, store: BlobStore, snapshot: WorkspaceShareSnapshot): Promise<TeamWorkspaceSummary | null> {
   return classify(db, store, JSON.stringify(['share', teamAppKey(snapshot.team)]), snapshot.team, () => Promise.resolve(snapshot))
 }

package/server-managed/team-reports.ts

@@ -5,6 +5,7 @@ import { Buffer } from 'node:buffer'
 import { backfillFindingIds, reportEntries, stampSecurityGroups } from '@preventive/report'
 import { managedFindingSourcePaths, readManagedReport } from '../common/managed/report-content.ts'
 import { filterReportData, projectFinding } from '../common/managed/report-filter.ts'
+import { revalidationLinks, securityLinks } from '../common/security-links.js'
 import type { BlobStore } from './blob-store.ts'
 import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
 import { triageWireEntry } from './triage-response.ts'
@@ -104,18 +105,12 @@ async function buildTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: Tea
   }))
   for (const result of results) if (result.status === 'rejected') throw result.reason
   const globalLinks = snapshot.linkRevision ? await db.getEnabledLinkGroups(snapshot.linkRevision) : []
-  const edges = new Map<string, Set<string>>()
-  const connect = (a: string, b: string) => {
-    if (!edges.has(a)) edges.set(a, new Set())
-    edges.get(a)!.add(b)
-  }
-  // A star is enough for transitive security propagation, without a quadratic
-  // expansion for large links. Unknown IDs can connect known findings.
-  for (const ids of globalLinks.concat(reports.flatMap(report => linksOf(report.data) ?? []))) {
-    for (const id of ids.slice(1)) { connect(ids[0]!, id); connect(id, ids[0]!) }
-  }
   const groups = reports.flatMap(report => groupsOf(report.data).map(group => group.map(f => projectFinding(f, report.data as Finding)!)))
-  stampSecurityGroups(groups, { linkedIds: id => edges.get(id) ?? [] })
+  // Unknown IDs can connect known findings. App findings link to the inputs
+  // they revalidated in other reports, as the App view counts them.
+  stampSecurityGroups(groups, { linkedIds: securityLinks([
+    ...globalLinks, ...reports.flatMap(report => linksOf(report.data) ?? []), ...revalidationLinks(groups.flat()),
+  ]).linkedIds })
   const securityIds = new Set(groups.flat().filter(f => f['isSecurity'] === true).map(f => String(f['id'])))
   const allIds = new Set<string>(), visible: Visibility = new Map()
   for (let i = 0; i < reports.length; i++) {

package/server-managed/upstream-cache.ts

@@ -27,6 +27,9 @@ export interface UpstreamCacheStore {
   // Key → value for those of `keys` that are kept.
   getUpstreamCacheEntries(keys: string[]): Promise<Map<string, string>>
   setUpstreamCacheEntry(key: string, value: string, cachedAt: number): Promise<void>
+  // Each of `entries` (key, value), as setUpstreamCacheEntry keeps one, in one
+  // statement: the last of a key given twice.
+  setUpstreamCacheEntries(entries: [key: string, value: string][], cachedAt: number): Promise<void>
   // Removes those under `prefix` (no LIKE wildcard in it) kept before `before`;
   // how many.
   deleteUpstreamCacheEntries(prefix: string, before: number): Promise<number>
@@ -40,6 +43,8 @@ export function upstreamCacheMethods(db: ManagedSql): UpstreamCacheStore {
     ON CONFLICT (cache_key) DO UPDATE SET value = excluded.value, cached_at = excluded.cached_at
     WHERE managed_upstream_cache.cached_at <= excluded.cached_at`)
   const remove = db.prepare('DELETE FROM managed_upstream_cache WHERE cache_key LIKE ? AND cached_at < ?')
+  // setUpstreamCacheEntries's, by how many it keeps at once.
+  const setMany = new Map<number, ReturnType<ManagedSql['prepare']>>()
   return {
     async getUpstreamCacheEntry(key) {
       const row = await get.get(key) as { value: string } | undefined
@@ -53,6 +58,17 @@ export function upstreamCacheMethods(db: ManagedSql): UpstreamCacheStore {
     async setUpstreamCacheEntry(key, value, cachedAt) {
       await set.run(key, value, cachedAt)
     },
+    async setUpstreamCacheEntries(entries, cachedAt) {
+      const rows = [...new Map(entries)]
+      if (rows.length === 0) return
+      let many = setMany.get(rows.length)
+      if (!many) {
+        setMany.set(rows.length, many = db.prepare(`INSERT INTO managed_upstream_cache (cache_key, value, cached_at) VALUES ${rows.map(() => '(?, ?, ?)').join(', ')}
+          ON CONFLICT (cache_key) DO UPDATE SET value = excluded.value, cached_at = excluded.cached_at
+          WHERE managed_upstream_cache.cached_at <= excluded.cached_at`))
+      }
+      await many.run(...rows.flatMap(([key, value]) => [key, value, cachedAt]))
+    },
     async deleteUpstreamCacheEntries(prefix, before) {
       return Number((await remove.run(`${prefix}%`, before)).changes)
     },