@ai-development-environment/server 0.0.29 → 0.0.31

package/standalone/src/services/builds/artifact-cache.ts

@@ -0,0 +1,235 @@
+import "server-only";
+
+import { createHash, randomUUID } from "node:crypto";
+import {
+  mkdir,
+  readdir,
+  readFile,
+  rename,
+  rm,
+  stat,
+  utimes,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import {
+  cancelArtifactTransfer,
+  expectArtifactTransfer,
+} from "./artifact-transfer";
+import { getServerServices } from "@/services/server-services";
+
+export type MaterializedArtifact = {
+  path: string;
+  filename: string;
+  contentType: string;
+  size: number;
+  etag: string;
+};
+
+type Sidecar = {
+  filename: string;
+  contentType: string;
+  size: number;
+};
+
+// Overridable so the cache can be pointed at a volume with room for large
+// packages, rather than whatever the temporary directory happens to allow.
+const CACHE_DIRECTORY =
+  process.env.ARTIFACT_CACHE_DIRECTORY?.trim() ||
+  join(tmpdir(), "ade-build-artifact-cache");
+const MAX_CACHE_BYTES =
+  Number(process.env.ARTIFACT_CACHE_MAX_BYTES) || 5 * 1024 ** 3;
+const MAX_CACHE_AGE_MS = 6 * 60 * 60 * 1_000;
+
+const globalCache = globalThis as typeof globalThis & {
+  buildArtifactMaterializations?: Map<string, Promise<MaterializedArtifact>>;
+};
+const inFlight =
+  globalCache.buildArtifactMaterializations ??
+  (globalCache.buildArtifactMaterializations = new Map());
+
+/**
+ * Transfers are billed against wall-clock time on a link we do not control, so
+ * the ceiling scales with the artifact: two minutes of overhead plus roughly a
+ * second per megabyte, capped so a stalled agent cannot hold a request forever.
+ */
+function transferTimeout(sizeBytes: number | null): number {
+  return Math.min(30 * 60_000, 120_000 + (sizeBytes ?? 0) / 1_000);
+}
+
+function cacheKey(artifactId: string, identity: string): string {
+  return createHash("sha1").update(`${artifactId}:${identity}`).digest("hex");
+}
+
+async function readSidecar(path: string): Promise<Sidecar | null> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (!parsed || typeof parsed !== "object") return null;
+    const { filename, contentType, size } = parsed as Record<string, unknown>;
+    if (
+      typeof filename !== "string" ||
+      typeof contentType !== "string" ||
+      typeof size !== "number"
+    ) {
+      return null;
+    }
+    return { filename, contentType, size };
+  } catch {
+    return null;
+  }
+}
+
+async function readCached(key: string): Promise<MaterializedArtifact | null> {
+  const path = join(CACHE_DIRECTORY, key);
+  const sidecar = await readSidecar(`${path}.json`);
+  if (!sidecar) return null;
+  try {
+    const information = await stat(path);
+    if (!information.isFile() || information.size !== sidecar.size) return null;
+    const now = new Date();
+    await utimes(path, now, now).catch(() => {});
+    return { path, etag: `"${key}"`, ...sidecar };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Drops the least recently used entries once the cache exceeds its budget, and
+ * anything past the maximum age. Entries are several hundred megabytes each, so
+ * skipping this would fill the temporary directory within a handful of builds.
+ *
+ * `keep` is the entry the caller is about to serve. It is exempt so that an
+ * artifact larger than the whole budget is still returned rather than deleted
+ * out from under the response that just fetched it.
+ */
+async function prune(keep: string): Promise<void> {
+  let entries;
+  try {
+    entries = await readdir(CACHE_DIRECTORY);
+  } catch {
+    return;
+  }
+  const files: {
+    path: string;
+    size: number;
+    mtimeMs: number;
+    keep: boolean;
+  }[] = [];
+  for (const entry of entries) {
+    if (entry.endsWith(".json")) continue;
+    const path = join(CACHE_DIRECTORY, entry);
+    try {
+      const information = await stat(path);
+      if (!information.isFile()) continue;
+      files.push({
+        path,
+        size: information.size,
+        mtimeMs: information.mtimeMs,
+        keep: entry === keep,
+      });
+    } catch {
+      continue;
+    }
+  }
+  const expired = Date.now() - MAX_CACHE_AGE_MS;
+  files.sort((left, right) => left.mtimeMs - right.mtimeMs);
+  // The kept entry counts against the budget but is never a deletion candidate.
+  let total = files.reduce((sum, file) => sum + file.size, 0);
+  for (const file of files) {
+    if (file.keep) continue;
+    if (total <= MAX_CACHE_BYTES && file.mtimeMs >= expired) continue;
+    await rm(file.path, { force: true }).catch(() => {});
+    await rm(`${file.path}.json`, { force: true }).catch(() => {});
+    total -= file.size;
+  }
+}
+
+async function transfer(
+  buildId: string,
+  artifactId: string,
+  key: string,
+  sizeBytes: number | null,
+): Promise<MaterializedArtifact> {
+  const uploadId = randomUUID();
+  const services = getServerServices();
+  const build = await services.buildsService.getBuild(buildId);
+  if (!build?.agentId) throw new Error("Build agent is unavailable");
+
+  const timeoutMs = transferTimeout(sizeBytes);
+  try {
+    const [received] = await Promise.all([
+      expectArtifactTransfer(uploadId, build.agentId, timeoutMs),
+      services.buildsService.prepareArtifactDownload(
+        buildId,
+        artifactId,
+        uploadId,
+        timeoutMs,
+      ),
+    ]);
+    await mkdir(CACHE_DIRECTORY, { recursive: true, mode: 0o700 });
+    const path = join(CACHE_DIRECTORY, key);
+    // Both paths live under the temporary directory, so this rename is atomic
+    // and never crosses a filesystem boundary.
+    await rename(received.path, path);
+    const size = (await stat(path)).size;
+    const sidecar: Sidecar = {
+      // The agent chooses the filename — it appends .tar.gz when it packages a
+      // directory — so that decision has to survive into later cache hits.
+      filename: received.filename,
+      contentType: received.contentType,
+      size,
+    };
+    // Written after the rename so a torn write cannot leave a valid-looking entry.
+    await writeFile(`${path}.json`, JSON.stringify(sidecar), { mode: 0o600 });
+    // Awaited so the budget actually holds by the time the response is served;
+    // a directory scan is negligible next to the transfer that just finished.
+    await prune(key).catch(() => {});
+    return { path, etag: `"${key}"`, ...sidecar };
+  } catch (error) {
+    cancelArtifactTransfer(uploadId, error);
+    throw error;
+  }
+}
+
+/**
+ * Returns a local copy of a build artifact, fetching it from the agent only when
+ * it is not already cached.
+ *
+ * The agent holds the only copy, and every fetch costs a full upload of the
+ * whole file. iOS issues a probe plus several ranged requests per install, so
+ * serving those directly from the relay would re-transfer the package several
+ * times over. Caching first makes range requests answerable at all, and collapses
+ * concurrent requests for the same artifact into one transfer.
+ */
+export async function materializeArtifact(
+  buildId: string,
+  artifactId: string,
+): Promise<MaterializedArtifact> {
+  const services = getServerServices();
+  const artifact = await services.buildsService.artifactForInstall(
+    buildId,
+    artifactId,
+  );
+  if (!artifact) throw new Error("Build artifact not found");
+
+  const key = cacheKey(artifactId, artifact.checksum ?? artifact.createdAt);
+  const cached = await readCached(key);
+  if (cached) return cached;
+
+  const existing = inFlight.get(key);
+  if (existing) return existing;
+
+  const pending = transfer(
+    buildId,
+    artifactId,
+    key,
+    artifact.sizeBytes,
+  ).finally(() => {
+    inFlight.delete(key);
+  });
+  inFlight.set(key, pending);
+  return pending;
+}

package/standalone/src/services/builds/artifact-transfer.ts

@@ -31,12 +31,13 @@ const transfers =
 export function expectArtifactTransfer(
   uploadId: string,
   agentId: string,
+  timeoutMs = 170_000,
 ): Promise<ArtifactTransfer> {
   return new Promise((resolve, reject) => {
     const timeout = setTimeout(() => {
       transfers.delete(uploadId);
       reject(new Error("The agent did not upload the artifact in time"));
-    }, 170_000);
+    }, timeoutMs);
     timeout.unref();
     transfers.set(uploadId, { agentId, resolve, reject, timeout });
   });

package/standalone/src/services/builds/builds.service.test.ts

@@ -247,6 +247,61 @@ describe("BuildsService", () => {
     expect(createJob).not.toHaveBeenCalled();
   });
 
+  test("requires a report-capable agent for worktree coverage", async () => {
+    const legacyWorktree = {
+      ...worktree(),
+      baseBranchOverride: null,
+      codebase: { ...worktree().codebase, defaultBranch: "main" },
+    };
+    getPrismaClient.mockResolvedValue({
+      build: { findUnique: vi.fn().mockResolvedValue(null) },
+      worktree: { findUnique: vi.fn().mockResolvedValue(legacyWorktree) },
+    });
+    const createJob = vi.fn();
+
+    await expect(
+      new BuildsService(control(createJob)).startWorktreeCoverage({
+        worktreeId: "worktree-1",
+        configurationId: "configuration-1",
+        destination,
+        requestId: "coverage-1",
+      }),
+    ).rejects.toThrow("updated for worktree coverage");
+    expect(createJob).not.toHaveBeenCalled();
+  });
+
+  test("bounds worktree coverage history and excludes report payloads", async () => {
+    const findMany = vi.fn().mockResolvedValue([]);
+    getPrismaClient.mockResolvedValue({ buildReport: { findMany } });
+
+    await new BuildsService(control()).coverageHistory("worktree-1");
+
+    const query = findMany.mock.calls[0]![0];
+    expect(query).toMatchObject({
+      where: {
+        kind: "CODE_COVERAGE",
+        source: "WORKTREE",
+        build: { worktreeId: "worktree-1" },
+      },
+      take: 50,
+      select: {
+        summaryJson: true,
+        build: { select: { artifacts: expect.any(Object) } },
+      },
+    });
+    expect(query.select).not.toHaveProperty("dataJson");
+    expect(query).not.toHaveProperty("include");
+  });
+
+  test("does not materialize reports for build-list queries", async () => {
+    const findMany = vi.fn().mockResolvedValue([]);
+    getPrismaClient.mockResolvedValue({ build: { findMany } });
+
+    await new BuildsService(control()).builds();
+
+    expect(findMany.mock.calls[0]![0].include).not.toHaveProperty("reports");
+  });
+
   test("rebuilds with the original destination and selected settings", async () => {
     getPrismaClient.mockResolvedValue({
       build: {