@stage5/lumine 0.2.54 → 0.2.55

package/README.md

@@ -252,6 +252,13 @@ lumine sponsor duty start zero --provider codex \
   --model gpt-5.6-sol --effort max --service-tier priority
 ```
 
+Sponsor commands use the same browser-approval login as the rest of Lumine.
+When no saved CLI login exists, the command opens Twinkle, waits for the person
+using that browser to approve their own account, and then resumes. Any Twinkle
+account may authenticate this way, but login never grants sponsor duty:
+`sponsor duty start` still requires that exact account to have a canonical,
+server-approved sponsor profile.
+
 While that foreground process has a fresh server lease, Zero or Ciel exposes a
 Build Workshop in chat and users may see the named sponsor and canonical queue.
 With no live duty, the website stays in its ordinary chat state and shows none
@@ -383,10 +390,13 @@ includes `error.details.retryIdempotencyKey` so a partial attempt can be
 resumed with the exact generated key.
 
 Subject and queue listings use opaque, stable snapshot cursors. `--all`
-follows them automatically, saves a checkpoint after every canonical page,
-and records completed queue coverage in the run audit; `--resume` continues
-the exact same request. With `--all --json`, bounded progress goes to stderr so
-stdout remains one pipe-safe JSON value. Recommendation scans default to the previous completed
+follows them automatically, fsyncs each confirmed page to a private NDJSON
+candidate spool, saves only bounded cursor/boundary/count metadata in the
+checkpoint, and records completed queue coverage in the run audit; `--resume`
+verifies the confirmed spool prefix and continues the exact same request. The
+final JSON contract still contains the complete collection, streamed from the
+spool instead of accumulated in memory. With `--all --json`, bounded progress
+goes to stderr so stdout remains one pipe-safe JSON value. Recommendation scans default to the previous completed
 run's start boundary for at-least-once coverage. Use `--after` for an explicit
 timestamp or `--include-legacy`
 for an intentional all-history scan. Subject `--after` is inclusive and

package/lib/admin-workflows.js

@@ -1,7 +1,22 @@
-import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
+import { createHash, randomUUID } from "node:crypto";
+import {
+  chmodSync,
+  closeSync,
+  createReadStream,
+  fsyncSync,
+  lstatSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readSync,
+  renameSync,
+  truncateSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import readline from "node:readline";
 
 import { writeAdminJsonFile, readAdminJsonFile } from "./admin-news.js";
 import { requestJson } from "./http.js";
@@ -9,9 +24,14 @@ import { requestJson } from "./http.js";
 const MAX_BATCH_TARGET_FILE_BYTES = 2 * 1024 * 1024;
 const MAX_BATCH_TARGETS = 20_000;
 const MAX_AUTOMATIC_PAGES = 100_000;
-const MAX_ADMIN_CHECKPOINT_BYTES = 64 * 1024 * 1024;
-const ADMIN_PAGINATION_SCHEMA_VERSION = 2;
+const MAX_LEGACY_ADMIN_CHECKPOINT_BYTES = 64 * 1024 * 1024;
+const MAX_PAGINATION_CHECKPOINT_BYTES = 256 * 1024;
+const MAX_BATCH_CHECKPOINT_BYTES = 64 * 1024 * 1024;
+const ADMIN_PAGINATION_SCHEMA_VERSION = 3;
+const LEGACY_ADMIN_PAGINATION_SCHEMA_VERSION = 2;
 const ADMIN_BATCH_SCHEMA_VERSION = 2;
+const PAGINATED_RESULT_STORAGE = Symbol("lumineAdminPaginationStorage");
+const SPOOL_READ_BUFFER_BYTES = 64 * 1024;
 
 function validationError(message) {
   const error = new Error(message);
@@ -59,30 +79,522 @@ function defaultCheckpointPath({ runId, operationName }) {
   return path.join(os.tmpdir(), `lumine-admin-run-${runId}-${safeName}.json`);
 }
 
-function aggregatePageResult({ operation, pages, checkpointPath, state }) {
-  const last = pages[pages.length - 1] || { ok: true, status: "success", data: {} };
+function operationFingerprintValue({
+  schemaVersion,
+  options,
+  operation,
+  runId,
+}) {
+  return fingerprint({
+    workflowSchemaVersion: schemaVersion,
+    runId,
+    apiUrl: String(options.apiUrl || "").replace(/\/$/, ""),
+    name: operation.name,
+    path: pathWithCursor(operation.path, ""),
+    pagination: {
+      collectionKey: operation.pagination.collectionKey,
+      coverageQueue: operation.pagination.coverageQueue || null,
+      coverageMode: operation.pagination.coverageMode || null,
+      after: operation.pagination.after ?? null,
+      filters: operation.pagination.filters || {},
+    },
+  });
+}
+
+function candidateSpoolPath(checkpointPath) {
+  return `${checkpointPath}.candidates-${randomUUID()}.ndjson`;
+}
+
+function candidateSpoolBelongsToCheckpoint({ checkpointPath, spoolPath }) {
+  const resolvedCheckpoint = path.resolve(checkpointPath);
+  const resolvedSpool = path.resolve(String(spoolPath || ""));
+  const expectedPrefix = `${path.basename(resolvedCheckpoint)}.candidates-`;
+  const spoolName = path.basename(resolvedSpool);
+  const spoolIdentifier = spoolName.slice(
+    expectedPrefix.length,
+    -".ndjson".length,
+  );
+  return (
+    path.dirname(resolvedSpool) === path.dirname(resolvedCheckpoint) &&
+    spoolName.startsWith(expectedPrefix) &&
+    spoolName.endsWith(".ndjson") &&
+    /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i.test(
+      spoolIdentifier,
+    )
+  );
+}
+
+function createCandidateSpool(checkpointPath) {
+  const spoolPath = candidateSpoolPath(checkpointPath);
+  mkdirSync(path.dirname(spoolPath), { recursive: true });
+  const descriptor = openSync(spoolPath, "wx", 0o600);
+  closeSync(descriptor);
+  chmodSync(spoolPath, 0o600);
   return {
+    spoolPath,
+    spoolHash: createHash("sha256"),
+    spoolBytes: 0,
+    spoolSha256: createHash("sha256").digest("hex"),
+  };
+}
+
+function loadSupersededCandidateSpool(checkpointPath) {
+  let saved;
+  try {
+    saved = readAdminJsonFile(checkpointPath, "the previous pagination checkpoint", {
+      maxBytes: MAX_LEGACY_ADMIN_CHECKPOINT_BYTES,
+    });
+  } catch {
+    return null;
+  }
+  const spoolPath = path.resolve(String(saved?.spoolPath || ""));
+  if (
+    saved?.kind !== "admin-pagination" ||
+    Number(saved?.schemaVersion) !== ADMIN_PAGINATION_SCHEMA_VERSION ||
+    !candidateSpoolBelongsToCheckpoint({ checkpointPath, spoolPath })
+  ) {
+    return null;
+  }
+  try {
+    const fileState = lstatSync(spoolPath);
+    return fileState.isFile() && !fileState.isSymbolicLink() ? spoolPath : null;
+  } catch {
+    return null;
+  }
+}
+
+function appendCandidateItems({ spoolPath, spoolHash, items }) {
+  const lines = items.map((item) => {
+    const serialized = JSON.stringify(item);
+    if (serialized === undefined) {
+      throw validationError(
+        "The API returned a pagination candidate that cannot be serialized.",
+      );
+    }
+    return serialized;
+  });
+  if (lines.length === 0) {
+    return { addedBytes: 0, spoolSha256: spoolHash.copy().digest("hex") };
+  }
+  const payload = Buffer.from(`${lines.join("\n")}\n`, "utf8");
+  const descriptor = openSync(spoolPath, "a", 0o600);
+  try {
+    writeFileSync(descriptor, payload);
+    fsyncSync(descriptor);
+  } finally {
+    closeSync(descriptor);
+  }
+  spoolHash.update(payload);
+  return {
+    addedBytes: payload.byteLength,
+    spoolSha256: spoolHash.copy().digest("hex"),
+  };
+}
+
+function normalizeCheckpointCount(value, label) {
+  const normalized = Number(value);
+  if (!Number.isSafeInteger(normalized) || normalized < 0) {
+    throw validationError(`The pagination checkpoint has an invalid ${label}.`);
+  }
+  return normalized;
+}
+
+function normalizeFilterCount(value, label) {
+  const normalized = Number(value);
+  if (!Number.isSafeInteger(normalized) || normalized < 0) {
+    throw validationError(`Pagination has an invalid ${label}.`);
+  }
+  return normalized;
+}
+
+function normalizeClientFilterSummary(value) {
+  if (value === null || value === undefined) return null;
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw validationError("Pagination has invalid client-filter metadata.");
+  }
+  if (
+    !Array.isArray(value.contentTypes) ||
+    value.contentTypes.some(
+      (contentType) =>
+        typeof contentType !== "string" || !contentType.trim(),
+    )
+  ) {
+    throw validationError(
+      "Pagination has invalid client-filter content types.",
+    );
+  }
+  return {
+    contentTypes: value.contentTypes.map((contentType) => contentType.trim()),
+    excludedItems: normalizeFilterCount(
+      value.excludedItems,
+      "client-filter excludedItems count",
+    ),
+  };
+}
+
+function accumulateClientFilterSummary(previous, pageValue) {
+  const page = normalizeClientFilterSummary(pageValue);
+  if (!page) return previous;
+  if (!previous) return page;
+  if (
+    JSON.stringify(previous.contentTypes) !== JSON.stringify(page.contentTypes)
+  ) {
+    throw validationError(
+      "The client-side content filter changed while paging one canonical snapshot.",
+    );
+  }
+  return {
+    ...previous,
+    excludedItems: normalizeFilterCount(
+      previous.excludedItems + page.excludedItems,
+      "client-filter aggregate excludedItems count",
+    ),
+  };
+}
+
+function normalizeOperatorViewFilterSummary(value) {
+  if (value === null || value === undefined) return null;
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw validationError("Pagination has invalid operator-view metadata.");
+  }
+  const mode = String(value.mode || "").trim();
+  if (!mode) {
+    throw validationError("Pagination has an invalid operator-view mode.");
+  }
+  return {
+    mode,
+    excludedItems: normalizeFilterCount(
+      value.excludedItems,
+      "operator-view excludedItems count",
+    ),
+    unknownStateItems: normalizeFilterCount(
+      value.unknownStateItems,
+      "operator-view unknownStateItems count",
+    ),
+  };
+}
+
+function accumulateOperatorViewFilterSummary(previous, pageValue) {
+  const page = normalizeOperatorViewFilterSummary(pageValue);
+  if (!page) return previous;
+  if (!previous) return page;
+  if (previous.mode !== page.mode) {
+    throw validationError(
+      "The operator-view filter changed while paging one canonical snapshot.",
+    );
+  }
+  return {
+    ...previous,
+    excludedItems: normalizeFilterCount(
+      previous.excludedItems + page.excludedItems,
+      "operator-view aggregate excludedItems count",
+    ),
+    unknownStateItems: normalizeFilterCount(
+      previous.unknownStateItems + page.unknownStateItems,
+      "operator-view aggregate unknownStateItems count",
+    ),
+  };
+}
+
+function verifyCandidateSpool({
+  checkpointPath,
+  spoolPath,
+  spoolBytes,
+  spoolSha256,
+  candidateCount,
+  discardUnconfirmedTail,
+}) {
+  if (!candidateSpoolBelongsToCheckpoint({ checkpointPath, spoolPath })) {
+    throw validationError(
+      "The pagination checkpoint references an invalid candidate spool.",
+    );
+  }
+  const confirmedBytes = normalizeCheckpointCount(spoolBytes, "spoolBytes");
+  const confirmedCount = normalizeCheckpointCount(
+    candidateCount,
+    "candidateCount",
+  );
+  const expectedDigest = String(spoolSha256 || "").toLowerCase();
+  if (!/^[a-f0-9]{64}$/.test(expectedDigest)) {
+    throw validationError(
+      "The pagination checkpoint has an invalid candidate-spool digest.",
+    );
+  }
+  let fileState;
+  try {
+    fileState = lstatSync(spoolPath);
+  } catch {
+    throw validationError(`Could not read pagination candidates at ${spoolPath}.`);
+  }
+  if (!fileState.isFile() || fileState.isSymbolicLink()) {
+    throw validationError(
+      "The pagination candidate spool must be a regular private file.",
+    );
+  }
+  if (fileState.size < confirmedBytes) {
+    throw validationError(
+      "The pagination candidate spool is shorter than its confirmed checkpoint.",
+    );
+  }
+  if (fileState.size > confirmedBytes) {
+    if (!discardUnconfirmedTail) {
+      throw validationError(
+        "The pagination candidate spool changed after its checkpoint was confirmed.",
+      );
+    }
+    truncateSync(spoolPath, confirmedBytes);
+  }
+  chmodSync(spoolPath, 0o600);
+  const spoolHash = createHash("sha256");
+  const descriptor = openSync(spoolPath, "r");
+  const buffer = Buffer.allocUnsafe(SPOOL_READ_BUFFER_BYTES);
+  let remaining = confirmedBytes;
+  let lineCount = 0;
+  try {
+    while (remaining > 0) {
+      const bytesRead = readSync(
+        descriptor,
+        buffer,
+        0,
+        Math.min(buffer.length, remaining),
+        null,
+      );
+      if (bytesRead <= 0) {
+        throw validationError(
+          "The pagination candidate spool ended before its confirmed boundary.",
+        );
+      }
+      const chunk = buffer.subarray(0, bytesRead);
+      spoolHash.update(chunk);
+      for (let index = 0; index < bytesRead; index += 1) {
+        if (chunk[index] === 10) lineCount += 1;
+      }
+      remaining -= bytesRead;
+    }
+  } finally {
+    closeSync(descriptor);
+  }
+  if (lineCount !== confirmedCount) {
+    throw validationError(
+      "The pagination candidate spool count does not match its checkpoint.",
+    );
+  }
+  if (spoolHash.copy().digest("hex") !== expectedDigest) {
+    throw validationError(
+      "The pagination candidate spool does not match its confirmed checkpoint digest.",
+    );
+  }
+  return spoolHash;
+}
+
+function writePaginationCheckpoint(checkpointPath, state) {
+  const { resumed: _resumed, ...checkpoint } = state;
+  writeAdminJsonFile(
+    checkpointPath,
+    { ...checkpoint, updatedAt: new Date().toISOString() },
+    { privateFile: true, maxBytes: MAX_PAGINATION_CHECKPOINT_BYTES },
+  );
+}
+
+function paginationStorageFromResult(result) {
+  return result?.[PAGINATED_RESULT_STORAGE] || null;
+}
+
+export function getPaginatedResultStorage(result) {
+  const storage = paginationStorageFromResult(result);
+  return storage ? { ...storage } : null;
+}
+
+function attachPaginationStorage(result, storage) {
+  Object.defineProperty(result, PAGINATED_RESULT_STORAGE, {
+    configurable: false,
+    enumerable: false,
+    writable: false,
+    value: Object.freeze({ ...storage }),
+  });
+  return result;
+}
+
+function serializedObjectEntries(value, excludedKeys = new Set()) {
+  const entries = [];
+  for (const [key, entryValue] of Object.entries(value || {})) {
+    if (excludedKeys.has(key)) continue;
+    const serialized = JSON.stringify(entryValue);
+    if (serialized !== undefined) {
+      entries.push(`${JSON.stringify(key)}:${serialized}`);
+    }
+  }
+  return entries;
+}
+
+export async function forEachPaginatedResultItem(result, visit) {
+  const storage = paginationStorageFromResult(result);
+  if (!storage) {
+    throw validationError("The result does not contain spooled pagination data.");
+  }
+  verifyCandidateSpool({ ...storage, discardUnconfirmedTail: false });
+  const input = createReadStream(storage.spoolPath, { encoding: "utf8" });
+  const observedHash = createHash("sha256");
+  let observedBytes = 0;
+  input.on("data", (chunk) => {
+    observedHash.update(chunk, "utf8");
+    observedBytes += Buffer.byteLength(chunk, "utf8");
+  });
+  const lines = readline.createInterface({ input, crlfDelay: Infinity });
+  let index = 0;
+  try {
+    for await (const line of lines) {
+      if (!line) continue;
+      let item;
+      try {
+        item = JSON.parse(line);
+      } catch {
+        throw validationError(
+          "The pagination candidate spool contains invalid JSON.",
+        );
+      }
+      await visit(item, index);
+      index += 1;
+    }
+  } finally {
+    lines.close();
+    input.destroy();
+  }
+  if (index !== storage.candidateCount) {
+    throw validationError(
+      "The pagination candidate spool changed while it was being read.",
+    );
+  }
+  if (
+    observedBytes !== storage.spoolBytes ||
+    observedHash.digest("hex") !== storage.spoolSha256
+  ) {
+    throw validationError(
+      "The pagination candidate spool changed while it was being read.",
+    );
+  }
+}
+
+export async function writePaginatedResultJson({ result, write }) {
+  const storage = paginationStorageFromResult(result);
+  if (!storage) {
+    await write(`${JSON.stringify(result)}\n`);
+    return;
+  }
+  const outerEntries = serializedObjectEntries(result, new Set(["data"]));
+  const dataEntries = serializedObjectEntries(
+    result.data,
+    new Set([storage.collectionKey]),
+  );
+  let prefix = "{";
+  if (outerEntries.length > 0) prefix += outerEntries.join(",");
+  if (outerEntries.length > 0) prefix += ",";
+  prefix += `${JSON.stringify("data")}:{`;
+  if (dataEntries.length > 0) prefix += `${dataEntries.join(",")},`;
+  prefix += `${JSON.stringify(storage.collectionKey)}:[`;
+  await write(prefix);
+  let wroteItem = false;
+  await forEachPaginatedResultItem(result, async (item) => {
+    await write(`${wroteItem ? "," : ""}${JSON.stringify(item)}`);
+    wroteItem = true;
+  });
+  await write("]}}\n");
+}
+
+async function writePaginatedResultFile(filePath, result) {
+  const resolved = path.resolve(String(filePath || "").trim());
+  if (!String(filePath || "").trim()) {
+    throw validationError("An output file path is required.");
+  }
+  mkdirSync(path.dirname(resolved), { recursive: true });
+  const temporary = `${resolved}.tmp-${process.pid}-${randomUUID()}`;
+  let descriptor;
+  try {
+    descriptor = openSync(temporary, "wx", 0o600);
+    await writePaginatedResultJson({
+      result,
+      write: async (chunk) => writeFileSync(descriptor, chunk, "utf8"),
+    });
+    fsyncSync(descriptor);
+    closeSync(descriptor);
+    descriptor = undefined;
+    renameSync(temporary, resolved);
+    chmodSync(resolved, 0o600);
+  } catch (error) {
+    if (descriptor !== undefined) {
+      try {
+        closeSync(descriptor);
+      } catch {
+        // The failed write may already have closed the descriptor.
+      }
+    }
+    try {
+      unlinkSync(temporary);
+    } catch {
+      // Exclusive create or successful rename may leave nothing to clean.
+    }
+    throw error;
+  }
+  return resolved;
+}
+
+function aggregatePageResult({ operation, lastPage, checkpointPath, state }) {
+  const last = lastPage || { ok: true, status: "success", data: {} };
+  const lastData = { ...(last.data || {}) };
+  delete lastData[operation.pagination.collectionKey];
+  if (state.filterSummariesComplete) {
+    if (state.clientFilter) lastData.clientFilter = state.clientFilter;
+    else delete lastData.clientFilter;
+    if (state.operatorViewFilter) {
+      lastData.operatorViewFilter = state.operatorViewFilter;
+    } else {
+      delete lastData.operatorViewFilter;
+    }
+  } else {
+    // Legacy v2 checkpoints stored filtered candidates but not exact per-page
+    // exclusion counters. Do not present a last-page subtotal as a full-scan
+    // total after migrating one of those checkpoints.
+    delete lastData.clientFilter;
+    delete lastData.operatorViewFilter;
+  }
+  const candidateStorage = {
+    format: "ndjson",
+    path: state.spoolPath,
+    byteLength: state.spoolBytes,
+    sha256: state.spoolSha256,
+  };
+  const result = {
     ...last,
     data: {
-      ...(last.data || {}),
-      [operation.pagination.collectionKey]: state.items,
+      ...lastData,
       pagination: {
         ...(last.data?.pagination || {}),
         nextCursor: state.nextCursor,
         hasMore: !state.exhausted,
         exhausted: state.exhausted,
         after: state.after,
+        snapshotMaxId: state.snapshotMaxId,
         snapshotTimeStamp: state.snapshotTimeStamp,
       },
       scan: {
         pages: state.pages,
         scannedCount: state.scannedCount,
-        candidateCount: state.items.length,
+        candidateCount: state.candidateCount,
         checkpointPath,
         resumed: state.resumed,
+        filterSummariesComplete: state.filterSummariesComplete,
+        candidateStorage,
       },
     },
   };
+  return attachPaginationStorage(result, {
+    checkpointPath,
+    collectionKey: operation.pagination.collectionKey,
+    spoolPath: state.spoolPath,
+    spoolBytes: state.spoolBytes,
+    spoolSha256: state.spoolSha256,
+    candidateCount: state.candidateCount,
+  });
 }
 
 export async function runAutomaticPagination({
@@ -101,19 +613,29 @@ export async function runAutomaticPagination({
     options.adminCheckpoint ||
       defaultCheckpointPath({ runId, operationName: operation.name }),
   );
-  const operationFingerprint = fingerprint({
-    workflowSchemaVersion: ADMIN_PAGINATION_SCHEMA_VERSION,
+  const requestedOutputPath = String(options.adminOutput || "").trim();
+  if (options.adminOutput && !requestedOutputPath) {
+    throw validationError("An output file path is required.");
+  }
+  const outputPath = requestedOutputPath
+    ? path.resolve(requestedOutputPath)
+    : null;
+  if (outputPath === checkpointPath) {
+    throw validationError(
+      "The pagination checkpoint and result output must use different files.",
+    );
+  }
+  const operationFingerprint = operationFingerprintValue({
+    schemaVersion: ADMIN_PAGINATION_SCHEMA_VERSION,
+    options,
+    operation,
+    runId,
+  });
+  const legacyOperationFingerprint = operationFingerprintValue({
+    schemaVersion: LEGACY_ADMIN_PAGINATION_SCHEMA_VERSION,
+    options,
+    operation,
     runId,
-    apiUrl: String(options.apiUrl || "").replace(/\/$/, ""),
-    name: operation.name,
-    path: pathWithCursor(operation.path, ""),
-    pagination: {
-      collectionKey: operation.pagination.collectionKey,
-      coverageQueue: operation.pagination.coverageQueue || null,
-      coverageMode: operation.pagination.coverageMode || null,
-      after: operation.pagination.after ?? null,
-      filters: operation.pagination.filters || {},
-    },
   });
   let state = {
     schemaVersion: ADMIN_PAGINATION_SCHEMA_VERSION,
@@ -129,29 +651,188 @@ export async function runAutomaticPagination({
     snapshotTimeStamp: null,
     after: operation.pagination.after ?? null,
     boundariesConfirmed: false,
-    items: [],
+    candidateCount: 0,
+    spoolPath: "",
+    spoolBytes: 0,
+    spoolSha256: "",
+    filterSummariesComplete: true,
+    clientFilter: null,
+    operatorViewFilter: null,
     resumed: false,
     updatedAt: new Date().toISOString(),
   };
+  let spoolHash;
   if (options.adminResume) {
     const saved = readAdminJsonFile(
       checkpointPath,
       "the pagination checkpoint",
-      { maxBytes: MAX_ADMIN_CHECKPOINT_BYTES },
+      { maxBytes: MAX_LEGACY_ADMIN_CHECKPOINT_BYTES },
+    );
+    const savedSchemaVersion = Number(saved?.schemaVersion);
+    const savedPages = normalizeCheckpointCount(saved?.pages, "pages");
+    const savedScannedCount = normalizeCheckpointCount(
+      saved?.scannedCount,
+      "scannedCount",
     );
+    const baseMatches =
+      saved?.kind === "admin-pagination" &&
+      Number(saved?.runId) === runId &&
+      ((savedPages === 0 && saved?.boundariesConfirmed === false) ||
+        (savedPages > 0 && saved?.boundariesConfirmed === true));
+    const currentMatches =
+      savedSchemaVersion === ADMIN_PAGINATION_SCHEMA_VERSION &&
+      saved?.operationFingerprint === operationFingerprint;
+    const legacyMatches =
+      savedSchemaVersion === LEGACY_ADMIN_PAGINATION_SCHEMA_VERSION &&
+      saved?.operationFingerprint === legacyOperationFingerprint &&
+      Array.isArray(saved?.items) &&
+      savedPages > 0;
+    if (!baseMatches || (!currentMatches && !legacyMatches)) {
+      throw validationError(
+        "The checkpoint does not belong to this run and exact listing request.",
+      );
+    }
+    const savedNextCursor = String(saved?.nextCursor || "");
+    const savedExhausted = saved?.exhausted;
     if (
-      saved?.schemaVersion !== ADMIN_PAGINATION_SCHEMA_VERSION ||
-      saved?.kind !== "admin-pagination" ||
-      saved?.operationFingerprint !== operationFingerprint ||
-      Number(saved?.runId) !== runId ||
-      !Array.isArray(saved?.items) ||
-      saved?.boundariesConfirmed !== true
+      typeof savedExhausted !== "boolean" ||
+      (!savedExhausted && savedPages > 0 && !savedNextCursor) ||
+      (savedExhausted && savedNextCursor)
     ) {
       throw validationError(
-        "The checkpoint does not belong to this run and exact listing request.",
+        "The pagination checkpoint has inconsistent exhaustion metadata.",
       );
     }
-    state = { ...state, ...saved, resumed: true };
+    if (legacyMatches) {
+      const createdSpool = createCandidateSpool(checkpointPath);
+      spoolHash = createdSpool.spoolHash;
+      let spoolBytes = 0;
+      let spoolSha256 = createdSpool.spoolSha256;
+      try {
+        for (let index = 0; index < saved.items.length; index += 250) {
+          const appended = appendCandidateItems({
+            spoolPath: createdSpool.spoolPath,
+            spoolHash,
+            items: saved.items.slice(index, index + 250),
+          });
+          spoolBytes += appended.addedBytes;
+          spoolSha256 = appended.spoolSha256;
+        }
+        state = {
+          ...state,
+          nextCursor: savedNextCursor,
+          exhausted: savedExhausted,
+          pages: savedPages,
+          scannedCount: savedScannedCount,
+          snapshotMaxId: normalizeBoundary(
+            saved?.snapshotMaxId,
+            "snapshotMaxId",
+          ),
+          snapshotTimeStamp: normalizeBoundary(
+            saved?.snapshotTimeStamp,
+            "snapshotTimeStamp",
+          ),
+          after: normalizeBoundary(saved?.after, "after"),
+          boundariesConfirmed: true,
+          candidateCount: saved.items.length,
+          spoolPath: createdSpool.spoolPath,
+          spoolBytes,
+          spoolSha256,
+          filterSummariesComplete: false,
+          resumed: true,
+        };
+        writePaginationCheckpoint(checkpointPath, state);
+      } catch (error) {
+        try {
+          unlinkSync(createdSpool.spoolPath);
+        } catch {
+          // A failed migration may have created no durable candidate spool.
+        }
+        throw error;
+      }
+    } else {
+      const candidateCount = normalizeCheckpointCount(
+        saved?.candidateCount,
+        "candidateCount",
+      );
+      state = {
+        ...state,
+        nextCursor: savedNextCursor,
+        exhausted: savedExhausted,
+        pages: savedPages,
+        scannedCount: savedScannedCount,
+        snapshotMaxId:
+          savedPages > 0
+            ? normalizeBoundary(saved?.snapshotMaxId, "snapshotMaxId")
+            : null,
+        snapshotTimeStamp:
+          savedPages > 0
+            ? normalizeBoundary(
+                saved?.snapshotTimeStamp,
+                "snapshotTimeStamp",
+              )
+            : null,
+        after:
+          savedPages > 0
+            ? normalizeBoundary(saved?.after, "after")
+            : operation.pagination.after ?? null,
+        boundariesConfirmed: saved?.boundariesConfirmed,
+        candidateCount,
+        spoolPath: path.resolve(String(saved?.spoolPath || "")),
+        spoolBytes: normalizeCheckpointCount(saved?.spoolBytes, "spoolBytes"),
+        spoolSha256: String(saved?.spoolSha256 || "").toLowerCase(),
+        filterSummariesComplete:
+          savedPages === 0 || saved?.filterSummariesComplete === true,
+        clientFilter:
+          savedPages > 0 && saved?.filterSummariesComplete === true
+            ? normalizeClientFilterSummary(saved?.clientFilter)
+            : null,
+        operatorViewFilter:
+          savedPages > 0 && saved?.filterSummariesComplete === true
+            ? normalizeOperatorViewFilterSummary(saved?.operatorViewFilter)
+            : null,
+        resumed: true,
+      };
+      spoolHash = verifyCandidateSpool({
+        checkpointPath,
+        spoolPath: state.spoolPath,
+        spoolBytes: state.spoolBytes,
+        spoolSha256: state.spoolSha256,
+        candidateCount: state.candidateCount,
+        discardUnconfirmedTail: true,
+      });
+    }
+  } else {
+    const supersededSpoolPath = loadSupersededCandidateSpool(checkpointPath);
+    const createdSpool = createCandidateSpool(checkpointPath);
+    spoolHash = createdSpool.spoolHash;
+    state.spoolPath = createdSpool.spoolPath;
+    state.spoolBytes = createdSpool.spoolBytes;
+    state.spoolSha256 = createdSpool.spoolSha256;
+    try {
+      writePaginationCheckpoint(checkpointPath, state);
+    } catch (error) {
+      try {
+        unlinkSync(state.spoolPath);
+      } catch {
+        // The failed initialization may leave no spool to clean.
+      }
+      throw error;
+    }
+    if (supersededSpoolPath && supersededSpoolPath !== state.spoolPath) {
+      try {
+        unlinkSync(supersededSpoolPath);
+      } catch (error) {
+        reportProgress(
+          `Lumine admin ${operation.name}: could not remove superseded candidate spool ${supersededSpoolPath} (${error?.message || error}).`,
+        );
+      }
+    }
+  }
+  if (outputPath === path.resolve(state.spoolPath)) {
+    throw validationError(
+      "The pagination candidate spool and result output must use different files.",
+    );
   }
   const progressEnabled = options.json === true;
   if (progressEnabled) {
@@ -159,17 +840,17 @@ export async function runAutomaticPagination({
       `Lumine admin ${operation.name}: ${state.resumed ? "resuming" : "starting"} canonical scan; checkpoint ${checkpointPath}.`,
     );
   }
-  const pages = [];
+  let lastPage = null;
   while (!state.exhausted) {
     if (state.pages >= MAX_AUTOMATIC_PAGES) {
       throw validationError(
-        `Automatic pagination stopped at the ${MAX_AUTOMATIC_PAGES}-page safety ceiling. Resume after narrowing the request.`,
+        `Automatic pagination stopped at the ${MAX_AUTOMATIC_PAGES}-page safety ceiling. Start a narrower request with a new checkpoint.`,
       );
     }
     const requestPath = pathWithCursor(operation.path, state.nextCursor);
     const raw = await fetchPage(requestPath);
     const page = transformPage(raw);
-    pages.push(page);
+    lastPage = page;
     const data = page?.data || {};
     const pagination = data.pagination;
     if (!pagination || typeof pagination !== "object" || Array.isArray(pagination)) {
@@ -224,31 +905,52 @@ export async function runAutomaticPagination({
       state.after = requestedAfter ?? pageAfter;
       state.boundariesConfirmed = true;
     }
-    state.items.push(...items);
-    state.pages += 1;
-    state.scannedCount += normalizeScannedCount(
-      pagination.scannedCount,
-      items.length,
-    );
-    state.nextCursor = nextCursor;
-    state.exhausted = pagination.exhausted;
-    state.updatedAt = new Date().toISOString();
-    writeAdminJsonFile(checkpointPath, state, {
-      privateFile: true,
-      maxBytes: MAX_ADMIN_CHECKPOINT_BYTES,
+    const appended = appendCandidateItems({
+      spoolPath: state.spoolPath,
+      spoolHash,
+      items,
     });
+    const clientFilter = state.filterSummariesComplete
+      ? accumulateClientFilterSummary(state.clientFilter, data.clientFilter)
+      : null;
+    const operatorViewFilter = state.filterSummariesComplete
+      ? accumulateOperatorViewFilterSummary(
+          state.operatorViewFilter,
+          data.operatorViewFilter,
+        )
+      : null;
+    const nextState = {
+      ...state,
+      pages: state.pages + 1,
+      scannedCount:
+        state.scannedCount +
+        normalizeScannedCount(
+          pagination.scannedCount,
+          items.length,
+        ),
+      candidateCount: state.candidateCount + items.length,
+      spoolBytes: state.spoolBytes + appended.addedBytes,
+      spoolSha256: appended.spoolSha256,
+      clientFilter,
+      operatorViewFilter,
+      nextCursor,
+      exhausted: pagination.exhausted,
+      updatedAt: new Date().toISOString(),
+    };
+    writePaginationCheckpoint(checkpointPath, nextState);
+    state = nextState;
     if (
       progressEnabled &&
-      (pages.length === 1 || state.pages % 10 === 0 || state.exhausted)
+      (state.pages === 1 || state.pages % 10 === 0 || state.exhausted)
     ) {
       reportProgress(
-        `Lumine admin ${operation.name}: ${state.pages} page(s), ${state.scannedCount} row(s) scanned, ${state.items.length} candidate(s)${state.exhausted ? "; canonical snapshot exhausted." : "; continuing."}`,
+        `Lumine admin ${operation.name}: ${state.pages} page(s), ${state.scannedCount} row(s) scanned, ${state.candidateCount} candidate(s)${state.exhausted ? "; canonical snapshot exhausted." : "; continuing."}`,
       );
     }
   }
   const result = aggregatePageResult({
     operation,
-    pages,
+    lastPage,
     checkpointPath,
     state,
   });
@@ -259,18 +961,16 @@ export async function runAutomaticPagination({
       after: state.after,
       pages: state.pages,
       scannedCount: state.scannedCount,
-      candidateCount: state.items.length,
+      candidateCount: state.candidateCount,
       snapshotMaxId: state.snapshotMaxId,
       snapshotTimeStamp: state.snapshotTimeStamp,
       exhausted: state.exhausted,
       filters: operation.pagination.filters || {},
     });
   }
-  if (options.adminOutput) {
-    result.data.scan.outputPath = writeAdminJsonFile(options.adminOutput, result, {
-      privateFile: true,
-      maxBytes: MAX_ADMIN_CHECKPOINT_BYTES,
-    });
+  if (outputPath) {
+    result.data.scan.outputPath = outputPath;
+    await writePaginatedResultFile(outputPath, result);
   }
   return result;
 }
@@ -365,7 +1065,7 @@ export async function runBatchSkips({
   };
   if (options.adminResume) {
     const saved = readAdminJsonFile(checkpointPath, "the batch checkpoint", {
-      maxBytes: MAX_ADMIN_CHECKPOINT_BYTES,
+      maxBytes: MAX_BATCH_CHECKPOINT_BYTES,
     });
     if (
       saved?.schemaVersion !== ADMIN_BATCH_SCHEMA_VERSION ||
@@ -381,7 +1081,7 @@ export async function runBatchSkips({
   } else {
     writeAdminJsonFile(checkpointPath, state, {
       privateFile: true,
-      maxBytes: MAX_ADMIN_CHECKPOINT_BYTES,
+      maxBytes: MAX_BATCH_CHECKPOINT_BYTES,
     });
   }
   for (const target of targets) {
@@ -405,7 +1105,7 @@ export async function runBatchSkips({
     state.updatedAt = new Date().toISOString();
     writeAdminJsonFile(checkpointPath, state, {
       privateFile: true,
-      maxBytes: MAX_ADMIN_CHECKPOINT_BYTES,
+      maxBytes: MAX_BATCH_CHECKPOINT_BYTES,
     });
   }
   const result = {
@@ -427,7 +1127,7 @@ export async function runBatchSkips({
   if (options.adminOutput) {
     result.data.batch.outputPath = writeAdminJsonFile(options.adminOutput, result, {
       privateFile: true,
-      maxBytes: MAX_ADMIN_CHECKPOINT_BYTES,
+      maxBytes: MAX_BATCH_CHECKPOINT_BYTES,
     });
   }
   return result;

package/lib/admin.js

@@ -1,4 +1,5 @@
 import { createHash, randomUUID } from "node:crypto";
+import { once } from "node:events";
 import { readFileSync } from "node:fs";
 import { assertAuthScope, resolveAuth } from "./auth.js";
 import { requestJson } from "./http.js";
@@ -8,7 +9,13 @@ import {
   validateNewsEditorial,
   writeNewsClaimArtifacts,
 } from "./admin-news.js";
-import { runAutomaticPagination, runBatchSkips } from "./admin-workflows.js";
+import {
+  forEachPaginatedResultItem,
+  getPaginatedResultStorage,
+  runAutomaticPagination,
+  runBatchSkips,
+  writePaginatedResultJson,
+} from "./admin-workflows.js";
 import {
   parseBuildReviewReceipt,
   runManagedBuildReview,
@@ -380,9 +387,25 @@ export function assertRecommendationWindowResult({ operation, result }) {
   }
 }
 
-function finishAdminOutput({ options, operation, result }) {
+async function finishAdminOutput({ options, operation, result }) {
+  const paginationStorage = getPaginatedResultStorage(result);
   if (options.json) {
-    console.log(JSON.stringify(result));
+    if (paginationStorage) {
+      await writePaginatedResultJson({
+        result,
+        write: async (chunk) => {
+          if (!process.stdout.write(chunk)) {
+            await once(process.stdout, "drain");
+          }
+        },
+      });
+    } else {
+      console.log(JSON.stringify(result));
+    }
+    return result;
+  }
+  if (paginationStorage) {
+    await printSpooledAdminResult({ result, storage: paginationStorage });
     return result;
   }
   printAdminResult({ operation, result });
@@ -2375,6 +2398,57 @@ function printAdminMonthlyMediaCosts(monthlyMediaCosts) {
   );
 }
 
+async function printSpooledAdminResult({ result, storage }) {
+  const data = result?.data || {};
+  const count = Number(storage.candidateCount || 0);
+  if (storage.collectionKey === "subjects") {
+    console.log(`${count} subject(s):`);
+    await forEachPaginatedResultItem(result, async (subject) => {
+      console.log(
+        `#${subject.id} ${subject.title || "(untitled)"} — ${subject.author?.username || "unknown"} — effort ${subject.effortLevel ?? "unknown"}`,
+      );
+      console.log(`  ${subject.url}`);
+    });
+  } else if (storage.collectionKey === "events") {
+    console.log(`${count} audit event(s):`);
+    await forEachPaginatedResultItem(result, async (event) => {
+      const target =
+        event.targetType && event.targetId
+          ? ` ${event.targetType}:${event.targetId}`
+          : "";
+      console.log(
+        `#${event.id} run ${event.runId ?? "-"} ${event.action}${target} — ${event.result}${event.changed === true ? " (changed)" : event.changed === false ? " (no change)" : ""}`,
+      );
+    });
+  } else if (storage.collectionKey === "comments") {
+    console.log(`${count} comment(s) for ${data.subject?.url || "this target"}:`);
+    await forEachPaginatedResultItem(result, async (comment) => {
+      console.log(
+        `#${comment.id} ${comment.author?.username || "unknown"}: ${comment.content || "(empty)"}`,
+      );
+    });
+  } else if (storage.collectionKey === "items") {
+    console.log(`${count} recommendation candidate(s):`);
+    await forEachPaginatedResultItem(result, async (item) => {
+      console.log(`#${item.contentId} ${item.contentType}`);
+      if (item.url || item.subjectUrl) {
+        console.log(`  ${item.url || item.subjectUrl}`);
+      }
+    });
+  } else if (storage.collectionKey === "builds") {
+    console.log(`${count} Build candidate(s):`);
+    await forEachPaginatedResultItem(result, async (build) => {
+      console.log(`#${build.id} ${build.title || "(untitled)"}`);
+      if (build.url) console.log(`  ${build.url}`);
+    });
+  } else {
+    console.log(
+      `${count} ${storage.collectionKey} candidate(s) saved at ${storage.spoolPath}.`,
+    );
+  }
+  printPagination(data.pagination);
+}
+
 function printAdminResult({ operation, result }) {
   const data = result?.data || {};
   if (Array.isArray(data.applications)) {

package/lib/auth.js

@@ -6,6 +6,9 @@ import { requestJson } from "./http.js";
 import { sleep, trimTrailingSlash } from "./util.js";
 
 export async function login(options) {
+  const report = options.loginProgressToStderr
+    ? (...args) => console.error(...args)
+    : (...args) => console.log(...args);
   const start = await requestJson({
     method: "POST",
     url: `${options.apiUrl}/cli/device/start`,
@@ -23,15 +26,15 @@ export async function login(options) {
   });
 
   const approvalUrl = start.verificationUriComplete || start.verificationUri;
-  console.log("Connect Lumine CLI to Twinkle.");
+  report("Connect Lumine CLI to Twinkle.");
   if (options.openBrowser && approvalUrl) {
-    console.log("Opening Twinkle in your browser...");
+    report("Opening Twinkle in your browser...");
     const opened = await openBrowser(approvalUrl);
-    if (!opened) console.log("Could not open the browser automatically.");
+    if (!opened) report("Could not open the browser automatically.");
   }
-  console.log(`Approval link: ${approvalUrl}`);
-  console.log(`Code: ${start.userCode}`);
-  console.log("Leave this terminal open. Waiting for approval...");
+  report(`Approval link: ${approvalUrl}`);
+  report(`Code: ${start.userCode}`);
+  report("Leave this terminal open. Waiting for approval...");
 
   const intervalMs = Math.max(Number(start.interval || 3), 1) * 1000;
   const startedAt = Date.now();
@@ -52,10 +55,10 @@ export async function login(options) {
       userId: tokenResponse.user?.id || null,
       expiresAt: Date.now() + Number(tokenResponse.expiresIn || 0) * 1000,
     });
-    console.log(
+    report(
       `Logged in${tokenResponse.user?.username ? ` as ${tokenResponse.user.username}` : ""}.`,
     );
-    console.log("You can now run `lumine` to choose a project.");
+    report("You can now run `lumine` to choose a project.");
     return;
   }
 
@@ -112,7 +115,10 @@ export async function ensureAuth(options) {
   } catch (error) {
     if (options.authToken || !isMissingLoginError(error)) throw error;
   }
-  await login(options);
+  await login({
+    ...options,
+    loginProgressToStderr: Boolean(options.json),
+  });
   return await resolveAuth(options);
 }
 

package/lib/commands.js

@@ -2907,7 +2907,7 @@ Options:
   --since-run           Recommendations since the previous completed run (default)
   --include-legacy      Explicitly include the full historical recommendation queue
   --all                 Automatically read every page in the canonical snapshot
-  --checkpoint <file>   Persist automatic pagination or batch progress
+  --checkpoint <file>   Persist bounded pagination metadata or batch progress
   --resume              Resume the exact request recorded by --checkpoint
   --output <file>       Save an admin result or newspaper claim as JSON
   --output-dir <dir>    Directory for managed Build review evidence

package/lib/sponsor.js

@@ -6,7 +6,7 @@ import { stdin as input, stdout as output } from "node:process";
 
 import { agentCommand } from "./agent.js";
 import { notifyBuildOwnerOfContribution } from "./api.js";
-import { resolveAuth, writeAuthFile } from "./auth.js";
+import { ensureAuth, resolveAuth, writeAuthFile } from "./auth.js";
 import { readCompleteBuildForumSnapshot } from "./forum.js";
 import { requestJson } from "./http.js";
 import { sleep } from "./util.js";
@@ -24,7 +24,7 @@ export async function sponsorCommand(options, commandServices) {
     .trim()
     .toLowerCase();
   if (area === "agreement") {
-    const auth = await resolveAuth(options);
+    const auth = await ensureAuth(options);
     const agreement = await sponsorRequest({
       options,
       auth,
@@ -50,7 +50,7 @@ export async function sponsorCommand(options, commandServices) {
     return;
   }
   if (area === "jobs") {
-    const auth = await resolveAuth(options);
+    const auth = await ensureAuth(options);
     const result = await sponsorRequest({
       options,
       auth,
@@ -67,7 +67,7 @@ export async function sponsorCommand(options, commandServices) {
 }
 
 async function applyToSponsor(options) {
-  const auth = await resolveAuth(options);
+  const auth = await ensureAuth(options);
   const agreement = await sponsorRequest({ options, auth, path: "/agreement" });
   if (!options.json) {
     printSponsorAgreement(agreement, { ...options, json: false });
@@ -135,7 +135,7 @@ async function applyToSponsor(options) {
 }
 
 async function withdrawSponsorApplication(options) {
-  const auth = await resolveAuth(options);
+  const auth = await ensureAuth(options);
   if (!options.assumeYes && input.isTTY && output.isTTY) {
     const rl = readline.createInterface({ input, output });
     try {
@@ -162,7 +162,7 @@ async function withdrawSponsorApplication(options) {
 }
 
 async function updateSponsorCapacity(options) {
-  const auth = await resolveAuth(options);
+  const auth = await ensureAuth(options);
   const status = await sponsorRequest({ options, auth, path: "/status" });
   if (!status.sponsor) {
     throw new Error("An approved sponsor account is required first.");
@@ -216,8 +216,8 @@ async function sponsorDutyCommand(options, args, commandServices) {
     return;
   }
   const persona = normalizePersona(args[1] || options.sponsorPersona);
-  const auth = await resolveAuth(options);
   if (["pause", "resume", "stop"].includes(action)) {
+    const auth = await ensureAuth(options);
     const state =
       action === "resume" ? "active" : action === "stop" ? "stopped" : "paused";
     const result = await sponsorRequest({
@@ -242,6 +242,7 @@ async function sponsorDutyCommand(options, args, commandServices) {
       "--service-tier is currently supported only by the Codex duty adapter.",
     );
   }
+  const auth = await ensureAuth(options);
   const started = await sponsorRequest({
     options,
     auth,
@@ -1134,7 +1135,7 @@ async function sponsorRequest({
 }
 
 async function printSponsorStatus(options) {
-  const auth = await resolveAuth(options);
+  const auth = await ensureAuth(options);
   const status = await sponsorRequest({ options, auth, path: "/status" });
   if (options.json) {
     console.log(JSON.stringify(status, null, 2));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stage5/lumine",
-  "version": "0.2.54",
+  "version": "0.2.55",
   "description": "Command line tools for launching Lumine builds on Twinkle.",
   "type": "module",
   "bin": {

package/sdk/LUMINE_ADMIN.md

@@ -881,12 +881,15 @@ feed-ID high-water mark and the server timestamp, then traverse the indexed
 row moved forward when it was reshared. Explicit legacy scans retain the
 descending primary-key walk. A page
 can be empty while `hasMore` remains true; continue until `exhausted`. `--all`
-does that automatically and writes a private checkpoint after every
-server-confirmed page; `--resume` continues only when the checkpoint belongs to
-the same API, run, and exact request. The final result can be copied to
-`--output`, while `--checkpoint` is resumable operational state. Subject
-`--after` is inclusive, and every opaque cursor is bound to its original
-filters.
+does that automatically, fsyncs each confirmed page to a private NDJSON
+candidate spool, and writes only bounded cursor, boundary, count, byte-length,
+and digest metadata to the private checkpoint. `--resume` continues only when
+the checkpoint belongs to the same API, run, and exact request and its confirmed
+spool prefix still matches those metadata; an unconfirmed fsynced tail is
+discarded before the next request. The final JSON collection is streamed from
+the spool and can be copied to a separate `--output` file, while `--checkpoint`
+remains resumable operational state. Subject `--after` is inclusive, and every
+opaque cursor is bound to its original filters.
 
 For `--all --json`, stdout remains exactly one JSON value. Scan-start, first-
 page, every-tenth-page, and exhaustion progress is written to **stderr** with