@stage5/lumine 0.2.67 → 0.2.69

package/lib/admin-workflows.js

@@ -130,7 +130,7 @@ function pathEntryExists(filePath) {
   }
 }
 
-function acquireCheckpointLock(checkpointPath, operationFingerprint) {
+export function acquireCheckpointLock(checkpointPath, operationFingerprint) {
   const lockPath = `${checkpointPath}.lock`;
   const reclaimPath = `${lockPath}.reclaim`;
   mkdirSync(path.dirname(lockPath), { recursive: true });
@@ -253,7 +253,7 @@ function removeDeadCheckpointLock(lockPath) {
   }
 }
 
-function releaseCheckpointLock(lock) {
+export function releaseCheckpointLock(lock) {
   try {
     const fileState = lstatSync(lock.lockPath);
     if (!fileState.isFile() || fileState.isSymbolicLink()) return;
@@ -282,6 +282,15 @@ function operationFingerprintValue({
     apiUrl: String(options.apiUrl || "").replace(/\/$/, ""),
     name: operation.name,
     path: pathWithCursor(operation.path, ""),
+    ...(operation.pagination.summaryKeys
+      ? { summaryKeys: operation.pagination.summaryKeys }
+      : {}),
+    // A completed pre-upgrade scan can still contain the old 30-day cap.
+    // Reject that checkpoint too: it would otherwise skip fetching entirely
+    // and record truncated coverage as a successful since-run scan.
+    ...(operation.pagination.coverageMode === "since-run"
+      ? { discoveryWindowContract: "stored-run-start-v1" }
+      : {}),
     ...(operation.name === "builds.candidates"
       ? {
           resultTransform: {
@@ -637,6 +646,16 @@ export async function forEachPaginatedResultItem(result, visit) {
       "The result does not contain spooled pagination data.",
     );
   }
+  if (storage.parts) {
+    let index = 0;
+    for (const part of storage.parts) {
+      await forEachPaginatedResultItem(
+        attachPaginationStorage({}, part),
+        async (item) => visit(item, index++),
+      );
+    }
+    return;
+  }
   verifyCandidateSpool({ ...storage, discardUnconfirmedTail: false });
   const input = createReadStream(storage.spoolPath, { encoding: "utf8" });
   const observedHash = createHash("sha256");
@@ -706,7 +725,7 @@ export async function writePaginatedResultJson({ result, write }) {
   await write("]}}\n");
 }
 
-async function writePaginatedResultFile(filePath, result) {
+export async function writePaginatedResultFile(filePath, result) {
   const resolved = path.resolve(String(filePath || "").trim());
   if (!String(filePath || "").trim()) {
     throw validationError("An output file path is required.");
@@ -743,9 +762,26 @@ async function writePaginatedResultFile(filePath, result) {
   return resolved;
 }
 
+// Keep large history event sets on disk when combining independently bounded
+// subject batches. Each part retains its own verified spool and snapshot.
+export function combinePaginatedResults(result, parts) {
+  const storages = parts.map(getPaginatedResultStorage);
+  if (
+    !storages.length ||
+    storages.some((entry) => !entry || entry.collectionKey !== storages[0].collectionKey)
+  ) {
+    throw validationError("Cannot combine incompatible paginated results.");
+  }
+  return attachPaginationStorage(result, {
+    collectionKey: storages[0].collectionKey,
+    candidateCount: storages.reduce((sum, entry) => sum + entry.candidateCount, 0),
+    parts: storages,
+  });
+}
+
 function aggregatePageResult({ operation, lastPage, checkpointPath, state }) {
   const last = lastPage || { ok: true, status: "success", data: {} };
-  const lastData = { ...(last.data || {}) };
+  const lastData = { ...(last.data || {}), ...state.summaryData };
   delete lastData[operation.pagination.collectionKey];
   const lastPagination = { ...(last.data?.pagination || {}) };
   const pageScannedCount = normalizeScannedCount(
@@ -809,6 +845,17 @@ function aggregatePageResult({ operation, lastPage, checkpointPath, state }) {
   });
 }
 
+function retainedSummaryData(operation, data) {
+  return Object.fromEntries(
+    (operation.pagination.summaryKeys || []).map((key) => {
+      if (!data || !Object.hasOwn(data, key)) {
+        throw validationError(`History scan is missing ${key} summary metadata. Start a fresh scan.`);
+      }
+      return [key, data[key]];
+    }),
+  );
+}
+
 export async function runAutomaticPagination(args) {
   const { options, operation, runId } = args;
   if (!operation.pagination) {
@@ -929,6 +976,7 @@ async function runAutomaticPaginationWithLock({
     filterSummariesComplete: true,
     clientFilter: null,
     operatorViewFilter: null,
+    summaryData: {},
     resumed: false,
     updatedAt: new Date().toISOString(),
   };
@@ -1059,6 +1107,7 @@ async function runAutomaticPaginationWithLock({
           savedPages > 0 && saved?.filterSummariesComplete === true
             ? normalizeOperatorViewFilterSummary(saved?.operatorViewFilter)
             : null,
+        summaryData: savedPages > 0 ? retainedSummaryData(operation, saved.summaryData) : {},
         resumed: true,
       };
       spoolHash = verifyCandidateSpool({
@@ -1215,6 +1264,7 @@ async function runAutomaticPaginationWithLock({
       spoolSha256: appended.spoolSha256,
       clientFilter,
       operatorViewFilter,
+      summaryData: retainedSummaryData(operation, data),
       nextCursor,
       exhausted: pagination.exhausted,
       updatedAt: new Date().toISOString(),