@preventive/triage 1.0.0-alpha.28 → 1.0.0-alpha.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/triage",
-  "version": "1.0.0-alpha.28",
+  "version": "1.0.0-alpha.29",
   "description": "Client & relay server for triaging of automated reports",
   "license": "MIT OR Apache-2.0",
   "author": {
@@ -97,6 +97,8 @@
     "server-managed/bundle-build.ts",
     "server-managed/bundle-build-leases.ts",
     "server-managed/bundle-build-worker.js",
+    "server-managed/bundle-build-diagnostics.js",
+    "server-managed/bundle-build-diagnostics.d.ts",
     "server-managed/bundle-catalog.ts",
     "server-managed/bundle-store.ts",
     "server-managed/bundle-summary-cache.ts",
@@ -261,6 +263,7 @@
     "@exodus/stasis-core": "1.0.0-beta.5",
     "@preventive/report": "1.0.0-alpha.4",
     "@preventive/upstream": "1.0.0-alpha.4",
+    "@vercel/functions": "^3.9.11",
     "ws": "^8.20.1"
   },
   "peerDependencies": {

package/server-managed/bundle-build-diagnostics.d.ts

@@ -0,0 +1,10 @@
+export interface BundleBuildDiagnostic {
+  message: string
+  name?: string
+  code?: string
+  status?: number
+  stack?: string
+  cause?: BundleBuildDiagnostic
+}
+export function bundleBuildDiagnostic(error: unknown, token: string | null): BundleBuildDiagnostic
+export type BundleBuildStage = 'worker-start' | 'build' | 'scope' | 'serialize' | 'compress'

package/server-managed/bundle-build-diagnostics.js

@@ -0,0 +1,23 @@
+// Whitelist diagnostic fields; never inspect/serialize an entire upstream
+// error, request, response, or workerData (which contains the GitHub token).
+export function bundleBuildDiagnostic(error, token, depth = 0) {
+  const redact = value => {
+    let text = String(value)
+    if (token) text = text.replaceAll(token, '[redacted]')
+    return text.replaceAll(/\b(?:github_pat_|gh[pousr]_)[\w]+/gu, '[redacted]')
+      .replaceAll(/(https?:\/\/)[^\s/@]+:[^\s/@]+@/gu, '$1[redacted]@')
+      .replaceAll(/([?&](?:access_token|token|key)=)[^\s&#]*/giu, '$1[redacted]')
+  }
+  if (!error || typeof error !== 'object') return { message: redact(error).slice(0, 2048) }
+  // Upstream HttpError messages append the response body after the status.
+  // Keep the method, URL and status, but omit that body from message and stack.
+  const raw = typeof error.message === 'string' ? error.message : 'Unknown error'
+  const message = error.name === 'HttpError' ? raw.replace(/(\s\d{3}):[\s\S]*$/u, '$1') : raw
+  const result = { message: redact(message).slice(0, 2048) }
+  if (typeof error.name === 'string') result.name = redact(error.name).slice(0, 128)
+  if (typeof error.code === 'string') result.code = redact(error.code).slice(0, 128)
+  if (typeof error.status === 'number') result.status = error.status
+  if (typeof error.stack === 'string') result.stack = redact(error.stack.replace(raw, message)).slice(0, 8192)
+  if (error.cause && depth < 3) result.cause = bundleBuildDiagnostic(error.cause, token, depth + 1)
+  return result
+}

package/server-managed/bundle-build-worker.js

@@ -6,39 +6,49 @@ import { buildGitHubBundle } from '@exodus/stasis/vfs-bundle'
 import { brotliOptions } from '@exodus/stasis-core/brotli'
 import { HttpError, createClient } from '@preventive/upstream/github.js'
 import { githubBundleFilename } from './bundle-build.ts'
+import { bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
 
-export async function buildStasisBundle({ input, github, token, maxBytes, scopes }, client = createClient({ token })) {
+export async function buildStasisBundle({ input, github, token, maxBytes, scopes }, client = createClient({ token }), progress = () => {}) {
   const allowed = path => scopes.some(scope => !scope || path === scope || path.startsWith(scope + '/'))
   // Stasis/upstream cache writes are disabled by default; this fresh worker
   // never enables a cache directory or inherits application credentials.
+  progress('build')
   const { bundle } = await buildGitHubBundle({
     github, sha: input.commit, directory: input.directory || undefined,
     entries: input.entries.map(entry => posix.relative(input.directory || '.', entry)),
     ...input.options, client,
   })
   const directory = bundle.repo?.directory ?? ''
+  progress('scope')
   // A workspace lockfile may widen the build root. Never publish it under a
   // narrower team grant: the storage location must cover the actual bundle.
   if (!allowed(directory)) throw new Error('build-scope')
+  progress('serialize')
   const serialized = bundle.serialize()
   if (Buffer.byteLength(serialized) > 200 * 1024 * 1024) throw new Error('too-large')
+  progress('compress')
   const bytes = brotliCompressSync(serialized, brotliOptions())
   if (bytes.length > maxBytes) throw new Error('too-large')
   return { bytes, directory, filename: githubBundleFilename(github, input.directory, input.commit) }
 }
 
 if (parentPort && workerData?.type === 'managed-bundle-build') {
-  try {
+  // Send diagnostics to the request's parent thread, which writes them before
+  // returning the HTTP response. Worker stdout can be lost on termination.
+  const send = message => {
     // Node worker messages have no browser target origin.
     // oxlint-disable-next-line unicorn/require-post-message-target-origin
-    parentPort.postMessage(await buildStasisBundle(workerData))
+    parentPort.postMessage(message)
+  }
+  try {
+    send(await buildStasisBundle(workerData, undefined, stage => send({ type: 'progress', stage })))
   } catch (error) {
+    const diagnostic = bundleBuildDiagnostic(error, workerData.token)
     // Do not return upstream bodies or paths outside the caller's grants.
-    const code = ['build-scope', 'too-large'].includes(error.message) ? error.message
+    const code = ['build-scope', 'too-large'].includes(diagnostic.message) ? diagnostic.message
       : error instanceof HttpError ? (error.status === 429 ? 'github-rate-limited' : 'github-build-failed')
-        : /no packageManager given|lockfile installs/u.test(error.message) ? 'build-lockfile'
+        : /no packageManager given|lockfile installs/u.test(diagnostic.message) ? 'build-lockfile'
           : 'build-failed'
-    // oxlint-disable-next-line unicorn/require-post-message-target-origin
-    parentPort.postMessage({ error: code, status: code === 'too-large' ? 413 : code === 'build-scope' ? 403 : 422 })
+    send({ error: code, status: code === 'too-large' ? 413 : code === 'build-scope' ? 403 : 422, diagnostic })
   }
 }

package/server-managed/bundle-build.ts

@@ -3,6 +3,7 @@ import { createHash, randomUUID } from 'node:crypto'
 import { posix } from 'node:path'
 import { Worker } from 'node:worker_threads'
 import { BUNDLE_BUILD_TIMEOUT_MS, type BundleBuildLeaseStore } from './bundle-build-leases.ts'
+import { type BundleBuildDiagnostic, type BundleBuildStage, bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
 
 export class BundleBuildError extends Error {
   status: number
@@ -113,28 +114,54 @@ export async function buildRepositoryBundle(userId: string, request: BuildReques
   signal.throwIfAborted()
   active.add(userId)
   let worker: Worker | undefined
+  let workerUrl: URL | undefined
+  let stage: BundleBuildStage = 'worker-start'
+  const buildId = randomUUID(), started = performance.now()
+  const context = { buildId, repoId: request.input.repoId, github: request.github, commit: request.input.commit,
+    directory: request.input.directory, entryCount: request.input.entries.length, node: process.version }
+  const log = (event: string, details: Record<string, unknown> = {}) => {
+    const write = event === 'failed' ? console.error : console.info
+    write('managed-bundle-build:', JSON.stringify({ ...context, workerUrl: workerUrl?.href, event, stage, elapsedMs: Math.round(performance.now() - started), ...details }))
+  }
   try {
-    worker = new Worker(new URL('./bundle-build-worker.js', import.meta.url), {
+    workerUrl = new URL('./bundle-build-worker.js', import.meta.url)
+    log('started')
+    worker = new Worker(workerUrl, {
       workerData: { ...request, type: 'managed-bundle-build' }, env: {}, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 512 },
     })
     const running = worker
     return await new Promise((resolve, reject) => {
-      const timer = setTimeout(() => reject(new BundleBuildError(504, 'build-timeout')), BUNDLE_BUILD_TIMEOUT_MS)
-      const abort = () => reject(new BundleBuildError(499, 'build-cancelled'))
+      let settled = false
+      const cleanup = () => { clearTimeout(timer); signal.removeEventListener('abort', abort) }
+      const fail = (status: number, code: string, details: Record<string, unknown> = {}) => {
+        if (settled) return
+        settled = true
+        cleanup()
+        log(code === 'build-cancelled' ? 'cancelled' : 'failed', { code, ...details })
+        reject(new BundleBuildError(status, code))
+      }
+      const timer = setTimeout(() => fail(504, 'build-timeout'), BUNDLE_BUILD_TIMEOUT_MS)
+      const abort = () => fail(signal.reason instanceof BundleBuildError ? signal.reason.status : 499,
+        signal.reason instanceof BundleBuildError ? signal.reason.code : 'build-cancelled')
       signal.addEventListener('abort', abort, { once: true })
-      running.once('message', (message: BuiltBundle & { error?: string; status?: number }) => {
-        clearTimeout(timer)
-        signal.removeEventListener('abort', abort)
-        if (message.error) reject(new BundleBuildError(message.status ?? 422, message.error))
-        else resolve({ ...message, bytes: Buffer.from(message.bytes) })
-      })
-      running.once('error', () => reject(new BundleBuildError(422, 'build-failed')))
-      running.once('exit', () => {
-        clearTimeout(timer)
-        signal.removeEventListener('abort', abort)
-        reject(new BundleBuildError(422, 'build-failed'))
+      running.on('message', (message: BuiltBundle & { type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
+        if (settled) return
+        if (message.type === 'progress' && message.stage) { stage = message.stage; log('progress'); return }
+        if (message.error) { fail(message.status ?? 422, message.error, { diagnostic: message.diagnostic }); return }
+        settled = true
+        cleanup()
+        log('completed', { byteSize: message.bytes.length })
+        resolve({ bytes: Buffer.from(message.bytes), directory: message.directory, filename: message.filename })
       })
+      running.once('error', error => fail(422, 'build-failed', { diagnostic: bundleBuildDiagnostic(error, request.token) }))
+      running.once('exit', exitCode => fail(422, 'build-failed', { exitCode }))
+      if (signal.aborted) abort()
     })
+  } catch (error) {
+    if (!(error instanceof BundleBuildError)) {
+      log('failed', { code: 'build-failed', diagnostic: bundleBuildDiagnostic(error, request.token) })
+    }
+    throw error
   } finally {
     try { await worker?.terminate() } finally { active.delete(userId) }
   }

package/server-managed/index.ts

@@ -88,8 +88,9 @@ async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOpti
   }
   async function handleRequest(req: Parameters<typeof routeRequest>[0], res: Parameters<typeof routeRequest>[1]): Promise<void> {
     const pendingMaintenance = automaticReap()
-    // Send the normal response without waiting for storage housekeeping, but
-    // keep the invocation alive until both finish. No detached serverless work.
+    // Send the normal response without waiting for storage housekeeping. The
+    // Vercel entrypoint registers this whole promise with waitUntil; awaiting
+    // work after res.end() alone cannot extend a serverless invocation.
     try { await routeRequest(req, res) }
     finally { await pendingMaintenance }
   }

package/server-managed/storage-maintenance.ts

@@ -134,6 +134,13 @@ interface StorageMigrationResult extends StorageEncryptionState {
   retryAt: number | null
   failures: { type: StorageMigrationRow['type']; id: string; message: string }[]
 }
+function rowFailure(row: StorageMigrationRow, message: string) {
+  const failure = { type: row.type, id: row.id, message }
+  // Emit before continuing: a later failure or terminated invocation must not
+  // hide earlier diagnostics. Never include provider errors or storage values.
+  console.warn('managed-storage-migration-row:', JSON.stringify(failure))
+  return failure
+}
 interface MigrationOptions { maxObjects?: number; maxMs?: number | undefined; signal?: AbortSignal | undefined }
 export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key: StorageKey,
   { maxObjects = 64, maxMs = 150_000, signal: stopping }: MigrationOptions = {}): Promise<StorageMigrationResult> {
@@ -152,9 +159,10 @@ export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key:
     if (rows.length === 0) { await db.advanceStorageMigration(state.cursor, null); break }
     for (const row of rows) {
       if (Date.now() >= deadline || signal.aborted) break
+      console.info('managed-storage-migration-row-start:', JSON.stringify({ type: row.type, id: row.id }))
       try {
         const pending = await migrateRow(raw, db, key, row, signal)
-        if (pending) failures.push({ type: row.type, id: row.id, message: pending })
+        if (pending) failures.push(rowFailure(row, pending))
       }
       catch (err) {
         // A row started late gets a full budget next time. Skip an oversized
@@ -163,8 +171,10 @@ export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key:
         const failure = signal.aborted
           ? `Could not migrate ${row.type} ${row.id} within MANAGED_STORAGE_ENCRYPTION_MIGRATE_MAX_MS=${maxMs}; increase the budget`
           : err instanceof StoragePayloadError ? err.message : null
-        if (failure === null) errors.push(new Error(`Could not migrate ${row.type} ${row.id}`, { cause: err }))
-        else failures.push({ type: row.type, id: row.id, message: failure })
+        if (failure === null) {
+          rowFailure(row, 'Migration operation failed')
+          errors.push(new Error(`Could not migrate ${row.type} ${row.id}`, { cause: err }))
+        } else failures.push(rowFailure(row, failure))
       }
       if (stopping?.aborted) break
       await db.advanceStorageMigration(state.cursor, row.position)

package/server-managed/storage.ts

@@ -41,8 +41,8 @@ export async function openManagedStorage(config: ManagedConfig) {
         if (!config.storageEncryptionMigrate || signal?.aborted) return
         const state = await db.getStorageEncryption()
         if (state?.complete && state.cleanupComplete) return
+        console.info('managed-storage-migration-start:', JSON.stringify({ migrated: state?.migrated, cursor: state?.cursor }))
         const result = await migrateStorage(raw, db, key!, { maxMs: config.storageEncryptionMigrateMaxMs, signal })
-        for (const failure of result.failures) console.warn('managed-storage-migration-row:', JSON.stringify(failure))
         console.info('managed-storage-migration:', JSON.stringify({
           complete: result.complete === 1, cleanupComplete: result.cleanupComplete === 1,
           migrated: result.migrated, cursor: result.cursor, retryAt: result.retryAt,