@vzn/vx 0.0.0 → 0.0.12

package/src/cache/layered-cache.ts

@@ -0,0 +1,335 @@
+// LayeredCache — composes the local cache with a remote HTTP cache.
+//
+// Read path:  try local. On miss, try remote; on remote hit, ingest
+// the artifact bytes into local so the next read is a local hit.
+//
+// Write path: write to local synchronously. Upload the local artifact
+// to remote as a fire-and-forget background task; failures log a
+// warning but never fail the user's run (the task already succeeded;
+// the only loss is the remote cache entry).
+//
+// The local and remote layers share the SAME artifact format — the
+// `<hash>.tar.zst` bytes ship across the wire verbatim. Metadata
+// (taskId, command, durationMs) travels separately: the caller
+// supplies it to `get()` via the `ctx` arg, and the remote layer
+// surfaces `durationMs` from its response. No stage dirs, no
+// meta.json, no tar.gz wrapping — the artifact is what it is.
+
+import type { Cache } from './cache.js'
+import type {
+  CacheEntry,
+  CacheGetContext,
+  CacheKeyInput,
+  CacheLayer,
+  CachePolicy,
+  CacheStats,
+  IngestMeta,
+  InvocationRecord,
+  OutputFileRow,
+  PruneOptions,
+  PruneResult,
+  RunRecord,
+  SaveArgs,
+} from './cache.js'
+import { FULL_CACHE_POLICY } from './cache.js'
+import type { RemoteCache } from './remote-cache.js'
+
+/**
+ * Cap on concurrent background PUTs. Keeps a burst of cache misses from
+ * opening one socket per task; excess uploads queue and drain FIFO.
+ */
+const UPLOAD_CONCURRENCY = 4
+
+export interface LayeredCacheOptions {
+  /** Called for remote-related errors that the layer suppresses. */
+  onRemoteError?: (err: Error) => void
+  /**
+   * The 4-axis read/write policy. The local slice (read/write) is
+   * already applied to the inner `Cache` by the caller; this layer reads
+   * `remoteRead` / `remoteWrite` to gate its OWN remote operations.
+   * Default: everything on.
+   */
+  policy?: CachePolicy
+}
+
+export class LayeredCache implements CacheLayer {
+  /**
+   * In-flight remote pulls keyed by hash. `prefetch` and `get` both go
+   * through here, so a key probed concurrently by both resolves a
+   * SINGLE remote GET. Each promise resolves `true` iff the artifact
+   * was successfully ingested into local. Entries are retained for the
+   * run's lifetime: a settled `false` records "remote already had no
+   * such artifact (or it was corrupt)", which lets `get` skip a second
+   * lazy probe of the same dead hash. The map is bounded by the number
+   * of distinct task keys in a run — negligible.
+   */
+  private readonly inflight = new Map<string, Promise<boolean>>()
+
+  /**
+   * Hashes whose local artifact was materialized FROM the remote layer
+   * this run (by `prefetch` or `get`'s read-through). A later `get`
+   * finds them as a local hit, but the work was still saved by the
+   * remote cache — so we flip `source` to `'remote'` and the
+   * orchestrator reports `cache-hit-remote`. Without this, a prefetch
+   * followed by a `get` would mislabel a genuine remote hit as local.
+   */
+  private readonly remoteSourced = new Set<string>()
+
+  /**
+   * Background write-through uploads. `save()` returns after the local
+   * write + byte capture; the PUT runs here so a cache-miss task never
+   * holds its scheduler worker slot for the upload round-trip. Bounded
+   * at UPLOAD_CONCURRENCY; `run()` awaits `drainUploads()` before
+   * `cache.close()`.
+   */
+  private readonly uploadQueue: Array<() => Promise<void>> = []
+  private activeUploads = 0
+  private drainWaiters: Array<() => void> = []
+
+  private readonly policy: CachePolicy
+
+  constructor(
+    private readonly local: Cache,
+    private readonly remote: RemoteCache,
+    private readonly options: LayeredCacheOptions = {},
+  ) {
+    this.policy = options.policy ?? FULL_CACHE_POLICY
+  }
+
+  async key(input: CacheKeyInput): Promise<string> {
+    return await this.local.key(input)
+  }
+
+  async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
+    // No-op when remote reads are off — there's nothing to warm from.
+    if (!this.policy.remoteRead) return false
+    return await this.pullFromRemote(hash, ctx)
+  }
+
+  async get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
+    const localHit = await this.local.get(hash, ctx)
+    if (localHit) {
+      // A prefetch this run may have materialized this entry FROM
+      // remote; the local row exists now, but the work was saved by
+      // the remote cache, so the provenance stays 'remote'.
+      return this.remoteSourced.has(hash) ? { ...localHit, source: 'remote' } : localHit
+    }
+
+    // Remote reads disabled (e.g. `--cache=remote:`): a local miss is a
+    // real miss; never touch the remote layer.
+    if (!this.policy.remoteRead) return null
+
+    // A prefetch may already have probed this hash (resolved) or be
+    // mid-flight. Awaiting the shared promise guarantees AT MOST ONE
+    // remote GET per key: a settled `false` means remote had nothing
+    // (no second probe), and a settled/in-flight `true` means the
+    // artifact was/will-be ingested locally — re-read below.
+    const ingested = await this.pullFromRemote(hash, ctx)
+    if (!ingested) return null
+
+    // The artifact is now in local, but this *lookup* was a remote
+    // hit — flip the source so callers can distinguish "saved work
+    // via the remote cache" from "saved work via a prior local run".
+    const materialized = await this.local.get(hash, ctx)
+    return materialized ? { ...materialized, source: 'remote' } : null
+  }
+
+  // Existence probe: local first, then a remote HEAD — no body
+  // transfer, no ingest, so the plan path stays read-only even against
+  // a remote cache. Remote errors degrade to a miss (never throw).
+  async has(hash: string): Promise<'local' | 'remote' | null> {
+    if ((await this.local.has(hash)) === 'local') return 'local'
+    if (!this.policy.remoteRead) return null
+    try {
+      return (await this.remote.has(hash)) ? 'remote' : null
+    } catch (err) {
+      this.reportRemoteError(err)
+      return null
+    }
+  }
+
+  /**
+   * Single implementation of the remote read-through, shared by
+   * `prefetch` and `get`. Idempotent per hash via `inflight`: the first
+   * caller starts the GET + validate + ingest; concurrent and later
+   * callers await the same promise. Resolves `true` when the artifact
+   * ends up in local, `false` on a remote miss / error / corruption
+   * (degrades to a cache miss; the error is reported, never thrown).
+   */
+  private pullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
+    const existing = this.inflight.get(hash)
+    if (existing) return existing
+    const p = this.doPullFromRemote(hash, ctx)
+    this.inflight.set(hash, p)
+    return p
+  }
+
+  private async doPullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
+    let remoteResult
+    try {
+      remoteResult = await this.remote.get(hash)
+    } catch (err) {
+      this.reportRemoteError(err)
+      return false
+    }
+    if (!remoteResult) return false
+
+    // Ingest the remote bytes into local using the caller-supplied
+    // taskId/command plus the remote-reported durationMs. The remote
+    // layer carries durationMs as an HTTP header (x-artifact-duration);
+    // taskId + command come from the orchestrator's TaskNode in scope.
+    // Without `ctx`, we can't populate a meaningful entries row, so
+    // ingest with placeholders — caller-side typing nudges everyone
+    // toward passing ctx.
+    const meta: IngestMeta = {
+      taskId: ctx?.taskId ?? `${hash}#unknown`,
+      command: ctx?.command ?? '',
+      durationMs: remoteResult.durationMs ?? 0,
+    }
+    try {
+      await this.local.ingest(hash, new Uint8Array(remoteResult.body), meta)
+    } catch (err) {
+      // The bytes came off the network — a corrupt/truncated remote
+      // artifact must degrade to a cache miss (task re-executes), not
+      // crash the run. Local-layer reads outside this block still
+      // propagate: local corruption is a real fault, not a network one.
+      this.reportRemoteError(err)
+      return false
+    }
+    this.remoteSourced.add(hash)
+    return true
+  }
+
+  outputsPath(hash: string): string {
+    return this.local.outputsPath(hash)
+  }
+
+  async hashFile(filePath: string): Promise<string> {
+    return await this.local.hashFile(filePath)
+  }
+
+  async restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
+    await this.local.restoreOutputs(hash, projectDir, workspaceRoot)
+  }
+
+  async save(args: SaveArgs): Promise<void> {
+    // Local write honors its own gate inside Cache.save (no-op when
+    // local writes are disabled).
+    await this.local.save(args)
+    if (!this.policy.remoteWrite) return
+    // Write-through upload, OFF the task's critical path. The bytes are
+    // captured NOW — read from the artifact the local layer just wrote
+    // (same format on both sides, no repacking), or, when local writes
+    // are disabled (`--cache=local:,remote:rw`), packed in memory while
+    // the output files are guaranteed still on disk. The PUT itself then
+    // runs in the bounded background pool so the task's worker slot is
+    // released immediately; `run()` awaits `drainUploads()` before
+    // closing the cache. Errors are logged, not propagated: the task
+    // already succeeded; we don't fail it on cache-server issues.
+    let bytes: Uint8Array
+    try {
+      bytes = this.local.localWritesEnabled
+        ? await Bun.file(this.local.outputsPath(args.hash)).bytes()
+        : await this.local.packArtifactBytes(args)
+    } catch (err) {
+      this.reportRemoteError(err)
+      return
+    }
+    const durationMs = args.entry.durationMs
+    this.enqueueUpload(async () => {
+      try {
+        await this.remote.put(args.hash, bytes, { durationMs })
+      } catch (err) {
+        this.reportRemoteError(err)
+      }
+    })
+  }
+
+  /**
+   * Resolves once every queued + in-flight background upload settles.
+   * `run()` calls this next to the prefetch drain, before
+   * `cache.close()` — an upload reading state after close would race.
+   */
+  async drainUploads(): Promise<void> {
+    if (this.activeUploads === 0 && this.uploadQueue.length === 0) return
+    await new Promise<void>((resolve) => this.drainWaiters.push(resolve))
+  }
+
+  private enqueueUpload(job: () => Promise<void>): void {
+    this.uploadQueue.push(job)
+    this.pumpUploads()
+  }
+
+  private pumpUploads(): void {
+    while (this.activeUploads < UPLOAD_CONCURRENCY && this.uploadQueue.length > 0) {
+      const job = this.uploadQueue.shift()!
+      this.activeUploads++
+      // Jobs never reject (each wraps its PUT in the never-fail guard).
+      void job().finally(() => {
+        this.activeUploads--
+        this.pumpUploads()
+      })
+    }
+    if (this.activeUploads === 0 && this.uploadQueue.length === 0 && this.drainWaiters.length > 0) {
+      const waiters = this.drainWaiters
+      this.drainWaiters = []
+      for (const w of waiters) w()
+    }
+  }
+
+  async ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void> {
+    await this.local.ingest(hash, compressed, meta)
+  }
+
+  loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
+    // Output-file fingerprints live in the local SQLite layer only —
+    // they describe the state on this machine's filesystem.
+    return this.local.loadOutputFilesBatch(hashes)
+  }
+
+  async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
+    return this.local.isOutputsCurrent(projectDir, expected)
+  }
+
+  recordRun(run: RunRecord): void {
+    this.local.recordRun(run)
+  }
+
+  recordRuns(runs: readonly RunRecord[]): void {
+    this.local.recordRuns(runs)
+  }
+
+  recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
+    this.local.recordRunBundle(bundle)
+  }
+
+  stats(): CacheStats {
+    return this.local.stats()
+  }
+
+  async prune(options: PruneOptions): Promise<PruneResult> {
+    return await this.local.prune(options)
+  }
+
+  close(): void {
+    this.local.close()
+  }
+
+  private reportRemoteError(err: unknown): void {
+    const e = err instanceof Error ? err : new Error(String(err))
+    // The remote cache is fully optional: NO remote failure — a 500, a
+    // network drop, a corrupt artifact, even a throwing onRemoteError
+    // callback — may ever fail the run. We report and degrade to a
+    // cache miss. The callback is guarded so a buggy reporter can't
+    // turn an optional-cache hiccup into a run failure.
+    try {
+      if (this.options.onRemoteError) {
+        this.options.onRemoteError(e)
+      } else {
+        process.stderr.write(`[vx] remote cache: ${e.message}\n`)
+      }
+    } catch {
+      // swallow — reporting must never escalate
+    }
+  }
+}

package/src/cache/remote-cache.ts

@@ -0,0 +1,251 @@
+// Remote cache HTTP client — speaks the Turborepo /v8/artifacts/ spec.
+//
+// Wire-level only. Knows nothing about local storage, tar formats, or
+// how the orchestrator uses the cache; those concerns live in the
+// LayeredCache. Three operations — GET (read), PUT (write), and HEAD
+// (existence probe for the plan path).
+//
+// Reference servers we want to interop with at the HTTP layer:
+//   ducktors/turborepo-remote-cache, Fox32/openturbo-remote-cache,
+//   Vercel hosted cache.
+
+import { createHmac, timingSafeEqual } from 'node:crypto'
+
+export interface RemoteCacheConfig {
+  /** Base URL of the cache server, e.g. https://cache.example.com */
+  baseUrl: string
+  /** Bearer token sent as `Authorization: Bearer <token>`. */
+  token: string
+  /** Optional Turbo-style tenant identifiers; sent as ?teamId=&slug=. */
+  teamId?: string
+  slug?: string
+  /** Per-request timeout in ms. Default 60_000. */
+  timeoutMs?: number
+  /**
+   * HMAC artifact-signing key (Turbo's `remoteCache.signature` scheme).
+   * When set, every PUT carries an `x-artifact-tag` over the artifact
+   * and every GET response must carry a matching tag — a missing or
+   * mismatched tag is a `RemoteCacheError`, never a silent accept.
+   */
+  signatureKey?: string
+  /**
+   * Untrusted per-PR partition id, sent as `x-vx-cache-scope`. A compatible
+   * remote-cache server MAY partition untrusted writes by this id so one fork
+   * PR can't read or poison another's cache. Ignored by trusted writes and by
+   * third-party Turbo servers.
+   */
+  cacheScope?: string
+}
+
+export interface RemotePutMetadata {
+  /** Task duration in ms. Sent as x-artifact-duration. */
+  durationMs: number
+}
+
+export interface RemoteGetResult {
+  body: ArrayBuffer
+  durationMs: number | undefined
+}
+
+/**
+ * Hard cap on a downloaded artifact body (compressed bytes). A malicious or
+ * compromised remote (or a MITM answering a GET) must not be able to stream an
+ * unbounded body into `res.arrayBuffer()` and exhaust the victim's memory —
+ * the download aborts past this and degrades to a cache miss. Mirrors the
+ * server-side PUT cap.
+ */
+export const MAX_REMOTE_ARTIFACT_BYTES = 512 * 1024 * 1024
+
+/**
+ * Read a response body, aborting once cumulative bytes exceed `max`. Streams
+ * via the body reader so a lying (or absent) content-length can't defeat the
+ * cap. Falls back to `arrayBuffer()` only when the body isn't a stream.
+ *
+ * Exported for tests: the `max` param lets a unit test exercise the
+ * mid-stream abort with a tiny cap instead of a 512 MiB body.
+ */
+export async function readBodyBounded(res: Response, max: number): Promise<ArrayBuffer> {
+  const reader = res.body?.getReader()
+  if (reader === undefined) return await res.arrayBuffer()
+  const chunks: Uint8Array[] = []
+  let total = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    if (value === undefined) continue
+    total += value.byteLength
+    if (total > max) {
+      await reader.cancel().catch(() => {})
+      throw new Error(`artifact exceeds ${max}-byte download cap`)
+    }
+    chunks.push(value)
+  }
+  const out = new Uint8Array(total)
+  let off = 0
+  for (const c of chunks) {
+    out.set(c, off)
+    off += c.byteLength
+  }
+  return out.buffer
+}
+
+export class RemoteCache {
+  constructor(private readonly config: RemoteCacheConfig) {}
+
+  async get(hash: string): Promise<RemoteGetResult | null> {
+    const res = await this.fetch('GET', this.artifactUrl(hash))
+    if (res.status === 404) return null
+    if (!res.ok) throw new RemoteCacheError(`GET ${hash} → ${res.status}`, res.status)
+    // Refuse an honestly-declared oversize body before reading a byte.
+    const declaredLen = Number(res.headers.get('content-length') ?? '')
+    if (Number.isFinite(declaredLen) && declaredLen > MAX_REMOTE_ARTIFACT_BYTES) {
+      throw new RemoteCacheError(
+        `GET ${hash} → artifact too large (${declaredLen} bytes > ${MAX_REMOTE_ARTIFACT_BYTES} cap)`,
+        res.status,
+      )
+    }
+    // A body that ends before the declared content-length (server died
+    // mid-transfer) throws from the reader, not from fetch() — wrap it here
+    // so callers see a typed RemoteCacheError and the layered cache can
+    // degrade it to a miss. The bounded read also aborts a body that lies
+    // about (or omits) its length once it crosses the cap.
+    let body: ArrayBuffer
+    try {
+      body = await readBodyBounded(res, MAX_REMOTE_ARTIFACT_BYTES)
+    } catch (err) {
+      throw new RemoteCacheError(
+        `GET ${hash} → body read failed: ${(err as Error).message}`,
+        res.status,
+        err,
+      )
+    }
+    if (this.config.signatureKey !== undefined) {
+      const received = res.headers.get('x-artifact-tag')
+      if (received === null) {
+        // A signing deployment must not silently accept unsigned
+        // artifacts — that would let an attacker strip the tag.
+        throw new RemoteCacheError(
+          `GET ${hash} → signature verification enabled but response carries no x-artifact-tag`,
+          res.status,
+        )
+      }
+      if (!this.tagMatches(hash, new Uint8Array(body), received)) {
+        throw new RemoteCacheError(`GET ${hash} → x-artifact-tag signature mismatch`, res.status)
+      }
+    }
+    return {
+      body,
+      durationMs: parseIntHeader(res.headers.get('x-artifact-duration')),
+    }
+  }
+
+  /**
+   * Existence probe — Turbo wire `HEAD /v8/artifacts/:hash` (200 =
+   * exists, 404 = not). No body transfer; the plan path (`--dry` /
+   * `--graph`) predicts remote hits with this so it never downloads
+   * artifacts. Other failures throw a `RemoteCacheError` like `get`;
+   * the LayeredCache degrades them to a miss.
+   */
+  async has(hash: string): Promise<boolean> {
+    const res = await this.fetch('HEAD', this.artifactUrl(hash))
+    if (res.status === 404) return false
+    if (!res.ok) throw new RemoteCacheError(`HEAD ${hash} → ${res.status}`, res.status)
+    return true
+  }
+
+  async put(hash: string, body: ArrayBuffer | Uint8Array, meta: RemotePutMetadata): Promise<void> {
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/octet-stream',
+      'Content-Length': String(body.byteLength),
+      'x-artifact-duration': String(meta.durationMs),
+    }
+    if (this.config.signatureKey !== undefined) {
+      headers['x-artifact-tag'] = this.artifactTag(
+        hash,
+        body instanceof Uint8Array ? body : new Uint8Array(body),
+      )
+    }
+
+    const res = await this.fetch('PUT', this.artifactUrl(hash), { body, headers })
+    // Any 2xx is success — Turbo-compatible servers answer 200/201/202.
+    if (!res.ok) {
+      throw new RemoteCacheError(`PUT ${hash} → ${res.status}`, res.status)
+    }
+  }
+
+  // Turbo's construction (crates/turborepo-cache/src/signature_authentication.rs):
+  // base64(HMAC-SHA256(key, utf8(hash) || utf8(teamId ?? '') || body)).
+  // Byte-compatible with Turbo so signing servers/clients interop.
+  private artifactTag(hash: string, body: Uint8Array): string {
+    return createHmac('sha256', this.config.signatureKey!)
+      .update(hash)
+      .update(this.config.teamId ?? '')
+      .update(body)
+      .digest('base64')
+  }
+
+  private tagMatches(hash: string, body: Uint8Array, received: string): boolean {
+    const expected = Buffer.from(this.artifactTag(hash, body), 'base64')
+    const actual = Buffer.from(received, 'base64')
+    return expected.length === actual.length && timingSafeEqual(expected, actual)
+  }
+
+  private artifactUrl(hash: string): string {
+    return this.urlWithTenancy(`${this.config.baseUrl}/v8/artifacts/${encodeURIComponent(hash)}`)
+  }
+
+  private urlWithTenancy(base: string): string {
+    const params = new URLSearchParams()
+    if (this.config.teamId) params.set('teamId', this.config.teamId)
+    if (this.config.slug) params.set('slug', this.config.slug)
+    const q = params.toString()
+    return q ? `${base}?${q}` : base
+  }
+
+  private async fetch(
+    method: string,
+    url: string,
+    init?: { body?: ArrayBuffer | Uint8Array | string; headers?: Record<string, string> },
+  ): Promise<Response> {
+    const timeoutMs = this.config.timeoutMs ?? 60_000
+    try {
+      return await fetch(url, {
+        method,
+        signal: AbortSignal.timeout(timeoutMs),
+        headers: {
+          Authorization: `Bearer ${this.config.token}`,
+          // The untrusted per-PR partition (an optional server-side extension;
+          // a third-party Turbo server ignores it). Isolates one fork PR's
+          // writes/reads from another's within the untrusted tier.
+          ...(this.config.cacheScope !== undefined
+            ? { 'x-vx-cache-scope': this.config.cacheScope }
+            : {}),
+          ...init?.headers,
+        },
+        ...(init?.body !== undefined ? { body: init.body } : {}),
+      })
+    } catch (err) {
+      if (err instanceof Error && (err.name === 'AbortError' || err.name === 'TimeoutError')) {
+        throw new RemoteCacheError(`${method} ${url} timed out after ${timeoutMs}ms`, 0, err)
+      }
+      throw new RemoteCacheError(`${method} ${url} failed: ${(err as Error).message}`, 0, err)
+    }
+  }
+}
+
+export class RemoteCacheError extends Error {
+  constructor(
+    message: string,
+    public readonly status: number,
+    public override readonly cause?: unknown,
+  ) {
+    super(message)
+    this.name = 'RemoteCacheError'
+  }
+}
+
+function parseIntHeader(v: string | null): number | undefined {
+  if (v === null) return undefined
+  const n = Number(v)
+  return Number.isFinite(n) ? n : undefined
+}