@stage5/lumine 0.2.68 → 0.2.69

package/lib/admin-featured-history.js

@@ -0,0 +1,185 @@
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { readAdminJsonFile, writeAdminJsonFile } from "./admin-news.js";
+import {
+  acquireCheckpointLock,
+  releaseCheckpointLock,
+  runAutomaticPagination,
+  combinePaginatedResults,
+  writePaginatedResultFile,
+} from "./admin-workflows.js";
+
+// API read bound, deliberately unrelated to the delegated addition policy.
+export const FEATURED_HISTORY_BATCH_SIZE = 100;
+
+function invalid(message) {
+  const error = new Error(message);
+  error.code = "CLI_ADMIN_CLI_VALIDATION";
+  return error;
+}
+
+export async function runBatchedFeaturedHistory(args) {
+  const { options, operation, runId } = args;
+  if (options.adminCursor) {
+    throw invalid("Use --resume with --all, not a single-batch --cursor.");
+  }
+  const subjectIds = operation.pagination.filters.subjectIds;
+  const fingerprint = createHash("sha256")
+    .update(
+      JSON.stringify({
+        version: 1,
+        runId,
+        apiUrl: options.apiUrl,
+        path: operation.path,
+      }),
+    )
+    .digest("hex");
+  const checkpointPath = path.resolve(
+    options.adminCheckpoint ||
+      path.join(
+        os.tmpdir(),
+        `lumine-admin-run-${runId}-featured-history-${fingerprint}.json`,
+      ),
+  );
+  const outputPath = options.adminOutput
+    ? path.resolve(options.adminOutput)
+    : null;
+  if (
+    outputPath === checkpointPath ||
+    outputPath?.startsWith(`${checkpointPath}.`)
+  ) {
+    throw invalid(
+      "History output must not overwrite its checkpoint or batch files.",
+    );
+  }
+  const lock = acquireCheckpointLock(checkpointPath, fingerprint);
+  try {
+    let generation;
+    if (options.adminResume) {
+      const saved = readAdminJsonFile(
+        checkpointPath,
+        "the Featured history batch checkpoint",
+      );
+      if (
+        saved.kind !== "featured-history-batches" ||
+        saved.fingerprint !== fingerprint ||
+        !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(
+          saved.generation || "",
+        )
+      ) {
+        throw invalid(
+          "The checkpoint does not belong to this run and exact history request.",
+        );
+      }
+      generation = saved.generation;
+    } else {
+      generation = randomUUID();
+      writeAdminJsonFile(
+        checkpointPath,
+        { kind: "featured-history-batches", fingerprint, generation },
+        { privateFile: true },
+      );
+    }
+    const results = [];
+    for (
+      let offset = 0;
+      offset < subjectIds.length;
+      offset += FEATURED_HISTORY_BATCH_SIZE
+    ) {
+      const ids = subjectIds.slice(
+        offset,
+        offset + FEATURED_HISTORY_BATCH_SIZE,
+      );
+      const batchIndex = offset / FEATURED_HISTORY_BATCH_SIZE;
+      const batchCheckpoint = `${checkpointPath}.${generation}.batch-${batchIndex}.json`;
+      const url = new URL(operation.path, "https://lumine.invalid");
+      url.searchParams.set("subjectIds", ids.join(","));
+      const result = await runAutomaticPagination({
+        ...args,
+        options: {
+          ...options,
+          adminOutput: undefined,
+          adminCheckpoint: batchCheckpoint,
+          adminResume: !!options.adminResume && existsSync(batchCheckpoint),
+        },
+        operation: {
+          ...operation,
+          path: `${url.pathname}${url.search}`,
+          pagination: { ...operation.pagination, filters: { subjectIds: ids } },
+        },
+      });
+      if (
+        JSON.stringify(result.data.subjects?.map((subject) => subject.id)) !==
+        JSON.stringify(ids)
+      ) {
+        throw invalid(
+          "The API did not return a canonical summary for every requested subject.",
+        );
+      }
+      results.push(result);
+    }
+    const firstCoverage = results[0].data.coverage;
+    const uniformCoverage = results.every(
+      (result) =>
+        result.data.coverage?.complete === firstCoverage.complete &&
+        result.data.coverage?.startedAt === firstCoverage.startedAt,
+    );
+    const result = combinePaginatedResults(
+      {
+        ok: true,
+        status: "success",
+        data: {
+          coverage: uniformCoverage
+            ? firstCoverage
+            : { complete: false, startedAt: null },
+          subjects: results.flatMap((result) => result.data.subjects),
+          pagination: {
+            nextCursor: null,
+            hasMore: false,
+            exhausted: true,
+            snapshotMaxId: null,
+            snapshotTimeStamp: null,
+            after: null,
+            snapshotScope: "per-batch",
+            eventOrder: "subject-batch-then-id-descending",
+          },
+          scan: {
+            checkpointPath,
+            resumed: !!options.adminResume,
+            pages: results.reduce(
+              (sum, result) => sum + result.data.scan.pages,
+              0,
+            ),
+            scannedCount: results.reduce(
+              (sum, result) => sum + result.data.scan.scannedCount,
+              0,
+            ),
+            candidateCount: results.reduce(
+              (sum, result) => sum + result.data.scan.candidateCount,
+              0,
+            ),
+            batches: results.map((result, index) => ({
+              subjectIds: subjectIds.slice(
+                index * FEATURED_HISTORY_BATCH_SIZE,
+                (index + 1) * FEATURED_HISTORY_BATCH_SIZE,
+              ),
+              coverage: result.data.coverage,
+              pagination: result.data.pagination,
+              scan: result.data.scan,
+            })),
+          },
+        },
+      },
+      results,
+    );
+    if (outputPath) {
+      result.data.scan.outputPath = outputPath;
+      await writePaginatedResultFile(outputPath, result);
+    }
+    return result;
+  } finally {
+    releaseCheckpointLock(lock);
+  }
+}

package/lib/admin-runtime-logs.js

@@ -141,7 +141,7 @@ function startIntentPath(options) {
   );
 }
 
-function readStartIntent(intentPath, apiUrl) {
+function readStartIntent(intentPath, apiUrl, host = "primary") {
   if (!existsSync(intentPath)) return null;
   const intent = readAdminJsonFile(
     intentPath,
@@ -156,6 +156,11 @@ function readStartIntent(intentPath, apiUrl) {
   ) {
     return null;
   }
+  if ((intent.host || "primary") !== host) {
+    throw validationError(
+      `A ${intent.host || "primary"} runtime-log start has an unresolved outcome. Replay that host's start before choosing ${host}; its request key was preserved.`,
+    );
+  }
   return intent;
 }
 
@@ -185,6 +190,7 @@ function createSession({ options, apiUrl, review }) {
     kind: "lumine-admin-runtime-log-review",
     apiUrl,
     reviewId: review.id,
+    ...(review.ownerHostId ? { ownerHostId: review.ownerHostId } : {}),
     leaseToken: review.leaseToken,
     status: review.status,
     outputDirectory,
@@ -581,10 +587,14 @@ export async function runAdminRuntimeLogWorkflow({
 }) {
   const apiUrl = normalizeApiUrl(options.apiUrl);
   if (operation.runtimeLogAction === "start") {
+    const host = operation.runtimeLogHost || "primary";
+    if (!["primary", "target"].includes(host)) {
+      throw validationError("Invalid runtime-log host.");
+    }
     const intentPath = startIntentPath(options);
     const intent = options.idempotencyKey
       ? null
-      : readStartIntent(intentPath, apiUrl);
+      : readStartIntent(intentPath, apiUrl, host);
     const startRequestId = intent?.requestId || requestId;
     if (!intent) {
       writeAdminJsonFile(
@@ -592,6 +602,7 @@ export async function runAdminRuntimeLogWorkflow({
         {
           kind: "lumine-admin-runtime-log-review-start",
           apiUrl,
+          host,
           requestId: startRequestId,
           createdAt: new Date().toISOString(),
         },
@@ -602,7 +613,7 @@ export async function runAdminRuntimeLogWorkflow({
     try {
       result = await requestJson({
         method: "POST",
-        url: `${apiUrl}/cli/admin/runtime-logs/reviews`,
+        url: `${apiUrl}/cli/admin/runtime-logs${operation.runtimeLogHost ? `/hosts/${host}` : ""}/reviews`,
         authToken,
         body: {},
         headers: requestHeaders({ requestId: startRequestId }),
@@ -631,6 +642,15 @@ export async function runAdminRuntimeLogWorkflow({
       throw error;
     }
     const review = validateReview(result?.data?.review);
+    if (
+      operation.runtimeLogHost &&
+      (!/^i-[a-f0-9]{17}$/.test(review.ownerHostId || "") ||
+        review.ownerHostRole !== host)
+    ) {
+      throw validationError(
+        "The API did not confirm a host-owned review. Preserve the start intent and recover it; do not clear logs.",
+      );
+    }
     const session = createSession({ options, apiUrl, review });
     // The session now holds the lease; a later rerun must start fresh.
     clearStartIntent(intentPath);

package/lib/admin-workflows.js

@@ -282,6 +282,15 @@ function operationFingerprintValue({
     apiUrl: String(options.apiUrl || "").replace(/\/$/, ""),
     name: operation.name,
     path: pathWithCursor(operation.path, ""),
+    ...(operation.pagination.summaryKeys
+      ? { summaryKeys: operation.pagination.summaryKeys }
+      : {}),
+    // A completed pre-upgrade scan can still contain the old 30-day cap.
+    // Reject that checkpoint too: it would otherwise skip fetching entirely
+    // and record truncated coverage as a successful since-run scan.
+    ...(operation.pagination.coverageMode === "since-run"
+      ? { discoveryWindowContract: "stored-run-start-v1" }
+      : {}),
     ...(operation.name === "builds.candidates"
       ? {
           resultTransform: {
@@ -637,6 +646,16 @@ export async function forEachPaginatedResultItem(result, visit) {
       "The result does not contain spooled pagination data.",
     );
   }
+  if (storage.parts) {
+    let index = 0;
+    for (const part of storage.parts) {
+      await forEachPaginatedResultItem(
+        attachPaginationStorage({}, part),
+        async (item) => visit(item, index++),
+      );
+    }
+    return;
+  }
   verifyCandidateSpool({ ...storage, discardUnconfirmedTail: false });
   const input = createReadStream(storage.spoolPath, { encoding: "utf8" });
   const observedHash = createHash("sha256");
@@ -706,7 +725,7 @@ export async function writePaginatedResultJson({ result, write }) {
   await write("]}}\n");
 }
 
-async function writePaginatedResultFile(filePath, result) {
+export async function writePaginatedResultFile(filePath, result) {
   const resolved = path.resolve(String(filePath || "").trim());
   if (!String(filePath || "").trim()) {
     throw validationError("An output file path is required.");
@@ -743,9 +762,26 @@ async function writePaginatedResultFile(filePath, result) {
   return resolved;
 }
 
+// Keep large history event sets on disk when combining independently bounded
+// subject batches. Each part retains its own verified spool and snapshot.
+export function combinePaginatedResults(result, parts) {
+  const storages = parts.map(getPaginatedResultStorage);
+  if (
+    !storages.length ||
+    storages.some((entry) => !entry || entry.collectionKey !== storages[0].collectionKey)
+  ) {
+    throw validationError("Cannot combine incompatible paginated results.");
+  }
+  return attachPaginationStorage(result, {
+    collectionKey: storages[0].collectionKey,
+    candidateCount: storages.reduce((sum, entry) => sum + entry.candidateCount, 0),
+    parts: storages,
+  });
+}
+
 function aggregatePageResult({ operation, lastPage, checkpointPath, state }) {
   const last = lastPage || { ok: true, status: "success", data: {} };
-  const lastData = { ...(last.data || {}) };
+  const lastData = { ...(last.data || {}), ...state.summaryData };
   delete lastData[operation.pagination.collectionKey];
   const lastPagination = { ...(last.data?.pagination || {}) };
   const pageScannedCount = normalizeScannedCount(
@@ -809,6 +845,17 @@ function aggregatePageResult({ operation, lastPage, checkpointPath, state }) {
   });
 }
 
+function retainedSummaryData(operation, data) {
+  return Object.fromEntries(
+    (operation.pagination.summaryKeys || []).map((key) => {
+      if (!data || !Object.hasOwn(data, key)) {
+        throw validationError(`History scan is missing ${key} summary metadata. Start a fresh scan.`);
+      }
+      return [key, data[key]];
+    }),
+  );
+}
+
 export async function runAutomaticPagination(args) {
   const { options, operation, runId } = args;
   if (!operation.pagination) {
@@ -929,6 +976,7 @@ async function runAutomaticPaginationWithLock({
     filterSummariesComplete: true,
     clientFilter: null,
     operatorViewFilter: null,
+    summaryData: {},
     resumed: false,
     updatedAt: new Date().toISOString(),
   };
@@ -1059,6 +1107,7 @@ async function runAutomaticPaginationWithLock({
           savedPages > 0 && saved?.filterSummariesComplete === true
             ? normalizeOperatorViewFilterSummary(saved?.operatorViewFilter)
             : null,
+        summaryData: savedPages > 0 ? retainedSummaryData(operation, saved.summaryData) : {},
         resumed: true,
       };
       spoolHash = verifyCandidateSpool({
@@ -1215,6 +1264,7 @@ async function runAutomaticPaginationWithLock({
       spoolSha256: appended.spoolSha256,
       clientFilter,
       operatorViewFilter,
+      summaryData: retainedSummaryData(operation, data),
       nextCursor,
       exhausted: pagination.exhausted,
       updatedAt: new Date().toISOString(),