@nebulacomponents/citable 1.18.2 → 1.20.0

package/src/commands/verifyRemediation.js

@@ -1,5 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { loadVerifiedRun } from '../shared/verifiedRunLoader.js';
 import crypto from 'node:crypto';
 import { readJson, writeJson, nowIso } from '../shared/io.js';
 import { validateAgainst } from '../shared/schemaValidator.js';
@@ -7,6 +8,7 @@ import { buildContext } from './context.js';
 import { runDetectors } from '../detectors/framework.js';
 import { selectDetectors } from '../detectors/index.js';
 import { remediateCommand } from './remediate.js';
+import { assessReobservation, resourceIdentityFor, sameResourceIdentity } from './compareSnapshots.js';
 
 const PKG_VERSION = JSON.parse(
   fs.readFileSync(new URL('../../package.json', import.meta.url), 'utf8')
@@ -20,14 +22,49 @@ function sha256(content) {
   return crypto.createHash('sha256').update(content, 'utf8').digest('hex');
 }
 
-function subjectKey(subject) {
-  return subject?.identifier || subject?.url || JSON.stringify(subject) || 'unknown';
+function findingIdentityKey(finding) {
+  const identity = resourceIdentityFor(finding);
+  const id = identity.resourceIds.size ? [...identity.resourceIds].sort().join(',')
+    : identity.effective || identity.requested || (identity.fallbackUrls.size === 1 ? [...identity.fallbackUrls][0] : JSON.stringify(finding?.subject || {}));
+  return `${finding?.detector_id || ''}|${id}`;
 }
 
 function newVerificationId() {
   return `VR-${nowIso().replace(/\D/g, '')}-${Math.random().toString(36).slice(2, 6)}`;
 }
 
+function canonicalJson(value) {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
+  if (value && typeof value === 'object') {
+    return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+export function recheckComparability(manifest, sourceFinding, afterFinding, ctx, detector) {
+  const sourceKind = manifest?.target?.kind || null;
+  const recheckKind = ctx.site?.mode || null;
+  const sourceDetectorVersion = sourceFinding?.provenance?.detector_version ?? 1;
+  const recheckDetectorVersion = afterFinding?.provenance?.detector_version ?? detector?.version ?? 1;
+  const sourceViewport = sourceFinding?.provenance?.viewport ?? null;
+  const recheckViewport = ctx.viewport ?? null;
+  const sourceConfig = manifest?.configuration_hash || null;
+  const recheckConfig = sha256(JSON.stringify(ctx.config));
+  const dimensions = {
+    tool_changed: (manifest?.tool_version || null) !== PKG_VERSION,
+    evaluator_changed: sourceDetectorVersion !== recheckDetectorVersion,
+    observation_method_changed: sourceKind !== recheckKind
+      || canonicalJson(sourceViewport) !== canonicalJson(recheckViewport),
+    configuration_changed: sourceConfig !== recheckConfig,
+  };
+  return {
+    comparable: !Object.values(dimensions).some(Boolean),
+    dimensions,
+    source: { tool_version: manifest?.tool_version || null, evaluator_version: sourceDetectorVersion, target_kind: sourceKind, configuration_hash: sourceConfig },
+    recheck: { tool_version: PKG_VERSION, evaluator_version: recheckDetectorVersion, target_kind: recheckKind, configuration_hash: recheckConfig },
+  };
+}
+
 /**
  * `citable verify remediation` — closed loop: source-run finding -> patch ->
  * detector re-run -> before/after evidence bundle.
@@ -37,7 +74,7 @@ function newVerificationId() {
  * never a silent `verified`.
  */
 export async function verifyRemediation(root, options = {}) {
-  const { run: runId, finding, target, apply = false, subject = null } = options;
+  const { run: runId, finding, target, apply = false, subject = null, viewport = null } = options;
   const runDir = path.join(root, '.citable', 'runs', runId || '');
   const limitations = [
     RESOLUTION_DEFINITION,
@@ -55,7 +92,7 @@ export async function verifyRemediation(root, options = {}) {
     recheck_run_id: null,
     detector_id: finding ? String(finding).toUpperCase() : null,
     subject: null,
-    verdict: { resolved: false, before_finding_ids: [], after_finding_ids: [], new_regressions: [], resolution_definition: RESOLUTION_DEFINITION },
+    verdict: { resolved: false, comparison_state: 'indeterminate', before_finding_ids: [], after_finding_ids: [], new_regressions: [], resolution_definition: RESOLUTION_DEFINITION },
     patch: null,
     provenance: { command: 'verify remediation', recheck_target: target || null, detectors_run: [], detectors_skipped: [], recheck_errors: [], recheck_warnings: [] },
     limitations,
@@ -73,23 +110,23 @@ export async function verifyRemediation(root, options = {}) {
     return finalizeResult(result);
   }
 
-  const beforeFindings = readJson(path.join(runDir, 'findings.json'));
-  const manifest = fs.existsSync(path.join(runDir, 'manifest.json'))
-    ? readJson(path.join(runDir, 'manifest.json'))
-    : null;
+  const source = loadVerifiedRun(runDir, { requireCompletedExecution: false, allowLegacy: true, requireCoverage: false });
+  const beforeFindings = source.findings;
+  const manifest = source.manifest;
   result.tool_version = manifest?.tool_version || PKG_VERSION;
   result.repository_commit = manifest?.repository_commit ?? null;
 
+  const requestedSubject = subject ? { subject: { identifier: subject } } : null;
   const matches = beforeFindings.filter((f) =>
     (f.detector_id || '').toUpperCase() === result.detector_id
-    && (!subject || subjectKey(f.subject) === subject));
+    && (!requestedSubject || sameResourceIdentity(f, requestedSubject)));
   if (matches.length === 0) {
     result.provenance.recheck_errors.push(`finding ${result.detector_id} not present in source run ${runId}`);
     result.status = 'blocked';
     return finalizeResult(result);
   }
   result.verdict.before_finding_ids = matches.map((f) => f.finding_id);
-  result.subject = { type: matches[0].subject?.type || 'unknown', identifier: subject || subjectKey(matches[0].subject) };
+  result.subject = { type: matches[0].subject?.type || 'unknown', identifier: subject || matches[0].subject?.identifier || matches[0].subject?.url || 'unknown' };
 
   // Optional gated patch application (production-safe pipeline from remediate.js)
   if (target && apply) {
@@ -133,7 +170,7 @@ export async function verifyRemediation(root, options = {}) {
   const namespace = result.detector_id.split('-')[0];
   let ctx;
   try {
-    ctx = await buildContext(root, { target: recheckTarget, baseUrl: options.baseUrl, refDate: options.refDate });
+    ctx = await buildContext(root, { target: recheckTarget, baseUrl: options.baseUrl, refDate: options.refDate, viewport });
   } catch (err) {
     result.provenance.recheck_errors.push(`re-check context failed: ${err.message}`);
     result.limitations.push('required_input: a readable target directory or URL for the detector re-run.');
@@ -155,24 +192,62 @@ export async function verifyRemediation(root, options = {}) {
 
   const afterMatches = afterFindings.filter((f) =>
     (f.detector_id || '').toUpperCase() === result.detector_id
-    && subjectKey(f.subject) === subjectKey(matches[0].subject));
+    && sameResourceIdentity(f, matches[0]));
   result.verdict.after_finding_ids = afterMatches.map((f) => f.finding_id);
 
-  const beforeKeys = new Set(beforeFindings.map((f) => `${f.detector_id}|${subjectKey(f.subject)}|${f.observation?.summary}`));
-  const afterKeys = new Set(afterFindings.map((f) => `${f.detector_id}|${subjectKey(f.subject)}|${f.observation?.summary}`));
+  const beforeKeys = new Set(beforeFindings.map((f) => `${findingIdentityKey(f)}|${f.observation?.summary}`));
+  const afterKeys = new Set(afterFindings.map((f) => `${findingIdentityKey(f)}|${f.observation?.summary}`));
   result.verdict.new_regressions = afterFindings
     .filter((f) => ['critical', 'high'].includes(f.classification?.severity))
-    .filter((f) => !beforeKeys.has(`${f.detector_id}|${subjectKey(f.subject)}|${f.observation?.summary}`))
+    .filter((f) => !beforeKeys.has(`${findingIdentityKey(f)}|${f.observation?.summary}`))
     .map((f) => ({ finding_id: f.finding_id, detector_id: f.detector_id, severity: f.classification.severity, summary: f.observation?.summary }));
 
+  const detectorDefinition = selectDetectors({ namespaces: [namespace] })
+    .find((candidate) => candidate.id === result.detector_id);
+  const comparability = recheckComparability(manifest, matches[0], afterMatches[0], ctx, detectorDefinition);
+
+  // Finalize the same collection envelope used by audit so detector absence
+  // is only resolution when the responsible resource was actually observed.
+  let recheckCoverage = null;
+  if (ctx.site?.finalizeCoverage) {
+    try { recheckCoverage = ctx.site.finalizeCoverage({ evaluated: true }); }
+    catch (error) { result.provenance.recheck_errors.push(`coverage finalization failed: ${error.message}`); }
+  }
+  let reobservation = assessReobservation(matches[0], recheckCoverage);
+  // Directory re-checks predate the coverage ledger and intentionally do not
+  // manufacture a package artifact. Their concrete page model is still a
+  // sufficient page-local observation when the subject is present.
+  if (!recheckCoverage && ctx.site?.pages?.some((page) => sameResourceIdentity(matches[0], page))) {
+    reobservation = { state: 'resolved', reason: 'page_present_in_recheck_model', resource: null };
+  }
+  result.comparison = {
+    state: comparability.comparable ? reobservation.state : 'not_comparable',
+    reason: reobservation.reason,
+    coverage_status: recheckCoverage?.coverage_status || 'not_available',
+    resource: reobservation.resource?.normalized_url || null,
+    comparability,
+  };
+  result.verdict.comparison_state = comparability.comparable ? reobservation.state : 'not_comparable';
+
   const errorsDuringRecheck = result.provenance.recheck_errors.length > 0;
   if (errorsDuringRecheck) {
-    result.status = 'blocked';
+    result.status = 'indeterminate';
+    result.verdict.resolved = false;
+    result.verdict.comparison_state = 'indeterminate';
+    result.limitations.push('Detector or collection errors occurred during the re-check; resolution cannot be established from a failed observation.');
+  } else if (!comparability.comparable) {
+    result.status = 'not_comparable';
     result.verdict.resolved = false;
-    result.limitations.push('Detector errors occurred during the re-check; resolution cannot be established from a failed observation.');
+    result.limitations.push('The source and re-check evaluator/tool/method/configuration envelopes are not comparable; resolution is not established.');
   } else if (afterMatches.length === 0) {
-    result.status = 'verified';
-    result.verdict.resolved = true;
+    if (reobservation.state === 'resolved') {
+      result.status = 'verified';
+      result.verdict.resolved = true;
+    } else {
+      result.status = reobservation.state === 'indeterminate' ? 'indeterminate' : 'not_reobserved';
+      result.verdict.resolved = false;
+      result.limitations.push(`The source subject was not sufficiently reobserved: ${reobservation.reason}.`);
+    }
   } else {
     result.status = 'not_resolved';
     result.verdict.resolved = false;

package/src/connectors/collectionResult.js

@@ -0,0 +1,80 @@
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+export const COLLECTION_SCHEMA_VERSION = 1;
+export const COLLECTION_COVERAGE_STATUSES = new Set([
+  'complete',
+  'truncated',
+  'indeterminate',
+  'provider_bounded',
+]);
+
+function nonNegativeInteger(value, name) {
+  if (!Number.isInteger(value) || value < 0) throw new TypeError(`${name} must be a non-negative integer`);
+  return value;
+}
+
+function stringList(value, name) {
+  if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) throw new TypeError(`${name} must be an array of strings`);
+  return [...new Set(value)];
+}
+
+/**
+ * Build the provider-neutral collection envelope. `items` are the items
+ * actually observed, never a provider-reported total or an inferred census.
+ */
+export function collectionResult({
+  items = [],
+  paginationState = {},
+  providerReportedTotal = null,
+  continuationState = null,
+  limitations = [],
+  errors = [],
+  providerCompleteness = 'declared',
+} = {}) {
+  if (!Array.isArray(items)) throw new TypeError('items must be an array');
+  if (!paginationState || typeof paginationState !== 'object' || Array.isArray(paginationState)) throw new TypeError('paginationState must be an object');
+  if (providerReportedTotal !== null) nonNegativeInteger(providerReportedTotal, 'providerReportedTotal');
+  stringList(limitations, 'limitations');
+  stringList(errors, 'errors');
+  if (continuationState !== null && (typeof continuationState !== 'object' || Array.isArray(continuationState))) throw new TypeError('continuationState must be an object or null');
+  if (!['declared', 'unknown'].includes(providerCompleteness)) throw new TypeError('providerCompleteness must be declared or unknown');
+
+  const totalErrors = providerReportedTotal !== null && providerReportedTotal < items.length
+    ? [`provider reported total ${providerReportedTotal}, below retrieved item count ${items.length}`]
+    : [];
+  const cleanErrors = [...new Set([...errors, ...totalErrors])];
+  const contradictoryTotal = providerReportedTotal !== null && providerReportedTotal < items.length;
+  const unresolvedTotal = providerReportedTotal !== null && providerReportedTotal > items.length && !continuationState;
+  const status = (cleanErrors.length || contradictoryTotal)
+    ? 'indeterminate'
+    : continuationState
+      ? 'truncated'
+      : (unresolvedTotal || providerCompleteness === 'unknown')
+        ? 'provider_bounded'
+        : 'complete';
+  const result = {
+    schema_version: COLLECTION_SCHEMA_VERSION,
+    items,
+    pagination_state: paginationState,
+    provider_reported_total: providerReportedTotal,
+    retrieved_total: items.length,
+    coverage_status: status,
+    continuation_state: continuationState,
+    limitations: [...new Set(limitations)],
+    errors: cleanErrors,
+  };
+  const check = validateAgainst('collection-result.schema.json', result);
+  if (!check.valid) throw new TypeError(`invalid collection result: ${check.errors.join('; ')}`);
+  return result;
+}
+
+export function paginationBoundary(context = {}) {
+  const value = context.collectionMaxPages ?? context.maxPages ?? 1000;
+  if (!Number.isInteger(value) || value < 1 || value > 10000) throw new RangeError('collection max pages must be an integer from 1 to 10000');
+  return value;
+}
+
+export function errorMessage(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  return message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]').slice(0, 300);
+}

package/src/connectors/ga4.js

@@ -1,4 +1,5 @@
 import { providerRequest } from './http.js';
+import { collectionResult, errorMessage, paginationBoundary } from './collectionResult.js';
 
 const DATA = 'https://analyticsdata.googleapis.com/v1beta';
 const ADMIN = 'https://analyticsadmin.googleapis.com/v1beta';
@@ -36,24 +37,58 @@ export const ga4Connector = {
     if (!dimensions.includes('landingPagePlusQueryString')) dimensions.push('landingPagePlusQueryString');
     const rows = [];
     const limitations = [];
+    const errors = [];
+    const items = [];
+    const maxPages = paginationBoundary(context);
+    let pagesRequested = 0;
+    let pagesRetrieved = 0;
+    let continuationState = null;
+    let providerTotal = null;
+    let connectorError = null;
     let offset = 0;
     do {
-      const result = await providerRequest(`${DATA}/properties/${encodeURIComponent(connection.property_id)}:runReport`, {
-        ...context, method: 'POST', body: {
-        dateRanges: [{ startDate, endDate }], dimensions: dimensions.map((name) => ({ name })),
-        metrics: metrics.map((metric) => ({ name: metric.external_name })), limit: '100000', offset: String(offset),
-        dimensionFilter: { filter: { fieldName: 'sessionDefaultChannelGroup', stringFilter: { matchType: 'EXACT', value: 'Organic Search' } } },
-        },
-      });
-      for (const row of result.rows || []) {
+      pagesRequested += 1;
+      let result;
+      try {
+        result = await providerRequest(`${DATA}/properties/${encodeURIComponent(connection.property_id)}:runReport`, {
+          ...context, method: 'POST', body: {
+          dateRanges: [{ startDate, endDate }], dimensions: dimensions.map((name) => ({ name })),
+          metrics: metrics.map((metric) => ({ name: metric.external_name })), limit: '100000', offset: String(offset),
+          dimensionFilter: { filter: { fieldName: 'sessionDefaultChannelGroup', stringFilter: { matchType: 'EXACT', value: 'Organic Search' } } },
+          },
+        });
+      } catch (error) {
+        connectorError ||= error;
+        errors.push(`offset ${offset}: ${errorMessage(error)}`);
+        continuationState = { offset };
+        break;
+      }
+      pagesRetrieved += 1;
+      const pageRows = result.rows || [];
+      items.push(...pageRows);
+      if (Number.isInteger(result.rowCount) && result.rowCount >= 0) providerTotal = result.rowCount;
+      for (const row of pageRows) {
         const values = Object.fromEntries(dimensions.map((name, index) => [name === 'landingPagePlusQueryString' ? 'url' : name, row.dimensionValues[index]?.value]));
         const date = values.date?.replace(/^(\d{4})(\d{2})(\d{2})$/, '$1-$2-$3');
         rows.push(...metrics.map((metric, index) => ({ metric, value: Number(row.metricValues[index]?.value), dimensions: values, observed_at: `${date}T00:00:00.000Z` })));
       }
       if (result.metadata?.dataLossFromOtherRow) limitations.push('Provider reports data loss from the (other) row.');
-      offset += (result.rows || []).length;
-      if (!result.rowCount || offset >= result.rowCount || !(result.rows || []).length) break;
+      offset += pageRows.length;
+      if ((result.rowCount === undefined || result.rowCount === null)
+        ? pageRows.length < 100000 || !pageRows.length
+        : offset >= result.rowCount || !pageRows.length) break;
+      if (pagesRequested >= maxPages) { continuationState = { offset }; break; }
     } while (true);
-    return { rows, cursor: endDate, limitations: [...new Set([...limitations, 'GA4 reporting identity, thresholding, attribution, and consent configuration affect results.', 'Sync is filtered to Organic Search sessions.'])] };
+    const providerCompleteness = limitations.some((item) => /data loss/i.test(item)) || providerTotal === null ? 'unknown' : 'declared';
+    const collection = collectionResult({
+      items,
+      paginationState: { pages_requested: pagesRequested, pages_retrieved: pagesRetrieved, boundary: { max_pages: maxPages } },
+      providerReportedTotal: providerTotal,
+      continuationState,
+      providerCompleteness,
+      limitations: [...new Set([...limitations, 'GA4 reporting identity, thresholding, attribution, and consent configuration affect results.', 'Sync is filtered to Organic Search sessions.'])],
+      errors,
+    });
+    return { rows, cursor: endDate, limitations: collection.limitations, collection, ...(connectorError ? { connectorError } : {}) };
   },
 };

package/src/connectors/gsc.js

@@ -1,4 +1,5 @@
 import { providerRequest } from './http.js';
+import { collectionResult, errorMessage, paginationBoundary } from './collectionResult.js';
 
 const BASE = 'https://www.googleapis.com/webmasters/v3';
 const METRICS = {
@@ -25,17 +26,51 @@ export const gscConnector = {
     if (!dimensions.includes('date')) dimensions.unshift('date');
     const rows = [];
     let startRow = 0;
+    const items = [];
+    const errors = [];
+    const maxPages = paginationBoundary(context);
+    let pagesRequested = 0;
+    let pagesRetrieved = 0;
+    let continuationState = null;
+    let providerTotal = null;
+    let connectorError = null;
     do {
-      const result = await providerRequest(`${BASE}/sites/${encodeURIComponent(connection.property_id)}/searchAnalytics/query`, {
-        ...context, method: 'POST', body: { startDate, endDate, dimensions, rowLimit: 25000, startRow, dataState: 'final' },
-      });
-      for (const row of result.rows || []) {
+      pagesRequested += 1;
+      let result;
+      try {
+        result = await providerRequest(`${BASE}/sites/${encodeURIComponent(connection.property_id)}/searchAnalytics/query`, {
+          ...context, method: 'POST', body: { startDate, endDate, dimensions, rowLimit: 25000, startRow, dataState: 'final' },
+        });
+      } catch (error) {
+        connectorError ||= error;
+        errors.push(`start row ${startRow}: ${errorMessage(error)}`);
+        continuationState = { start_row: startRow };
+        break;
+      }
+      pagesRetrieved += 1;
+      const pageRows = result.rows || [];
+      items.push(...pageRows);
+      if (Number.isInteger(result.rowCount) && result.rowCount >= 0) providerTotal = result.rowCount;
+      for (const row of pageRows) {
         const values = Object.fromEntries(dimensions.map((name, index) => [name === 'page' ? 'url' : name, row.keys[index]]));
         for (const metric of metrics) rows.push({ metric, value: row[metric.external_name], dimensions: values, observed_at: `${values.date}T00:00:00.000Z` });
       }
-      if ((result.rows || []).length < 25000) break;
+      if (pageRows.length < 25000) break;
       startRow += 25000;
+      if (pagesRequested >= maxPages) {
+        continuationState = { start_row: startRow };
+        break;
+      }
     } while (true);
-    return { rows, cursor: endDate, limitations: ['Search Console privacy filtering and aggregation apply.', 'Search Analytics does not guarantee every data row; the API can return top rows only.', 'Final data can still be revised by the provider.'] };
+    const collection = collectionResult({
+      items,
+      paginationState: { pages_requested: pagesRequested, pages_retrieved: pagesRetrieved, boundary: { max_pages: maxPages } },
+      providerReportedTotal: providerTotal,
+      continuationState,
+      providerCompleteness: 'unknown',
+      limitations: ['Search Console privacy filtering and aggregation apply.', 'Search Analytics does not guarantee every data row; the API can return top rows only.', 'Final data can still be revised by the provider.'],
+      errors,
+    });
+    return { rows, cursor: endDate, limitations: collection.limitations, collection, ...(connectorError ? { connectorError } : {}) };
   },
 };

package/src/connectors/index.js

@@ -4,6 +4,7 @@ import { wordpressConnector } from './wordpress.js';
 import { webflowConnector } from './webflow.js';
 import { indexnowConnector } from './indexnow.js';
 import { mcpConnector } from './mcp/pilot.js';
+export { collectionResult, paginationBoundary, errorMessage } from './collectionResult.js';
 
 const CONNECTORS = new Map([gscConnector, ga4Connector, wordpressConnector, webflowConnector, indexnowConnector, mcpConnector].map((connector) => [connector.provider, connector]));
 
@@ -18,4 +19,3 @@ export function listConnectors() {
 }
 
 export { McpClient, McpStdioTransport, McpHttpTransport, ALLOWED_MCP_SERVERS, isServerAllowlisted, isToolAllowlisted, assertToolAllowed, collectMcpEvidence, mcpConnector, diagnoseMcpTransports } from './mcp/index.js';
-

package/src/connectors/webflow.js

@@ -1,7 +1,9 @@
 import { providerRequest } from './http.js';
 import { sha256, nowIso } from '../shared/io.js';
+import { collectionResult, errorMessage, paginationBoundary } from './collectionResult.js';
 
 const BASE = 'https://api.webflow.com/v2';
+const API_ORIGIN = new URL(BASE).origin;
 
 const METRICS = {
   pages: { unit: 'count', value_type: 'integer' },
@@ -24,6 +26,96 @@ function computeWebflowPayloadHash(payload) {
   return sha256(JSON.stringify(canonical));
 }
 
+function safeContinuation(url, { preserveOrigin = true } = {}) {
+  try {
+    const parsed = new URL(url);
+    return { ...(preserveOrigin ? { url: `${parsed.origin}${parsed.pathname}` } : { path: parsed.pathname }), query_redacted: parsed.search.length > 0 };
+  } catch {
+    return { url: '[invalid continuation]', query_redacted: true };
+  }
+}
+
+async function collectWebflowList(url, field, context) {
+  const maxPages = paginationBoundary(context);
+  const items = [];
+  const errors = [];
+  const limitations = [];
+  let providerTotal = null;
+  let pagesRequested = 0;
+  let pagesRetrieved = 0;
+  let offset = 0;
+  let cursor = null;
+  let nextUrl = null;
+  let continuationState = null;
+  let providerCompleteness = 'declared';
+  let connectorError = null;
+  while (pagesRequested < maxPages) {
+    pagesRequested += 1;
+    const query = cursor
+      ? `?limit=100&cursor=${encodeURIComponent(cursor)}`
+      : `?limit=100${offset ? `&offset=${offset}` : ''}`;
+    let result;
+    try {
+      if (nextUrl) {
+        let continuation;
+        try { continuation = new URL(nextUrl); } catch {
+          errors.push(`${field} page ${pagesRequested}: malformed continuation URL`);
+          continuationState = { next: safeContinuation(nextUrl), reason: 'invalid_url' };
+          break;
+        }
+        if (continuation.origin !== API_ORIGIN || continuation.protocol !== 'https:') {
+          errors.push(`${field} page ${pagesRequested}: continuation origin is outside the Webflow API`);
+          continuationState = { next: safeContinuation(nextUrl, { preserveOrigin: false }), reason: 'origin_mismatch' };
+          break;
+        }
+      }
+      result = await providerRequest(nextUrl || `${url}${query}`, context);
+      const pageItems = Array.isArray(result?.[field]) ? result[field] : [];
+      pagesRetrieved += 1;
+      items.push(...pageItems);
+      const pagination = result.pagination || {};
+      const reported = result.total ?? pagination.total ?? pagination.totalCount ?? null;
+      if (Number.isInteger(reported) && reported >= 0) providerTotal = reported;
+      const next = result.nextCursor ?? pagination.nextCursor ?? result.next?.cursor ?? null;
+      const providerNextUrl = typeof result.next === 'string' ? result.next : (typeof pagination.next === 'string' ? pagination.next : null);
+      const nextOffset = result.nextOffset ?? pagination.nextOffset ?? null;
+      nextUrl = null;
+      if (next !== null && (typeof next !== 'string' || !next.trim())) {
+        errors.push(`${field} page ${pagesRequested}: malformed continuation cursor`);
+        continuationState = { cursor: next };
+        break;
+      }
+      if (providerNextUrl !== null && (!/^https:\/\//i.test(providerNextUrl))) {
+        errors.push(`${field} page ${pagesRequested}: malformed continuation URL`);
+        continuationState = { next: safeContinuation(providerNextUrl, { preserveOrigin: false }) };
+        break;
+      }
+      if (next) cursor = String(next);
+      else if (providerNextUrl) { nextUrl = providerNextUrl; cursor = null; }
+      else if (nextOffset !== null && (!Number.isInteger(nextOffset) || nextOffset <= offset)) {
+        errors.push(`${field} page ${pagesRequested}: malformed continuation offset`);
+        continuationState = { offset: nextOffset };
+        break;
+      }
+      else if (providerTotal !== null && items.length >= providerTotal) break;
+      else if (Number.isInteger(nextOffset) && nextOffset >= 0) { offset = nextOffset; cursor = null; }
+      else if (pageItems.length >= 100 && (providerTotal === null || items.length < providerTotal)) offset += pageItems.length;
+      else { providerCompleteness = 'unknown'; break; }
+      if (providerTotal !== null && items.length >= providerTotal) break;
+    } catch (error) {
+      connectorError ||= error;
+      errors.push(`${field} page ${pagesRequested}: ${errorMessage(error)}`);
+      continuationState = cursor ? { cursor } : nextUrl ? { next: safeContinuation(nextUrl) } : { offset };
+      break;
+    }
+  }
+  if (!continuationState && pagesRequested >= maxPages && (providerTotal === null || items.length < providerTotal)) {
+    continuationState = cursor ? { cursor } : nextUrl ? { next: safeContinuation(nextUrl) } : { offset: offset + 100 };
+    limitations.push(`Webflow ${field} collection reached the ${maxPages}-page boundary.`);
+  }
+  return { items, providerTotal, providerCompleteness, connectorError, pagesRequested, pagesRetrieved, continuationState, limitations, errors };
+}
+
 export const webflowConnector = {
   provider: 'webflow',
   defaultCredentialEnv: 'WEBFLOW_API_TOKEN',
@@ -62,10 +154,26 @@ export const webflowConnector = {
     const siteId = connection.property_id;
     const rows = [];
     const metricNames = new Set(metrics.map((m) => m.external_name));
+    const allItems = [];
+    const errors = [];
+    const limitations = [];
+    let pagesRequested = 0;
+    let pagesRetrieved = 0;
+    let continuationState = null;
+    let providerTotal = 0;
+    let providerTotalKnown = true;
+    let connectorError = null;
 
     if (metricNames.has('pages')) {
-      const pageResult = await providerRequest(`${BASE}/sites/${encodeURIComponent(siteId)}/pages`, context);
-      const pages = pageResult.pages || [];
+      const pageResult = await collectWebflowList(`${BASE}/sites/${encodeURIComponent(siteId)}/pages`, 'pages', context);
+      const pages = pageResult.items;
+      allItems.push(...pages.map((item) => ({ ...item, _collection_type: 'pages' })));
+      errors.push(...pageResult.errors); limitations.push(...pageResult.limitations);
+      pagesRequested += pageResult.pagesRequested; pagesRetrieved += pageResult.pagesRetrieved;
+      continuationState ||= pageResult.continuationState;
+      connectorError ||= pageResult.connectorError;
+      if (pageResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (pageResult.providerTotal === null) providerTotalKnown = false; else providerTotal += pageResult.providerTotal;
       const pageMetric = metrics.find((m) => m.external_name === 'pages');
       rows.push({
         metric: pageMetric,
@@ -76,8 +184,15 @@ export const webflowConnector = {
     }
 
     if (metricNames.has('collections') || metricNames.has('items')) {
-      const collectionsResult = await providerRequest(`${BASE}/sites/${encodeURIComponent(siteId)}/collections`, context);
-      const collections = collectionsResult.collections || [];
+      const collectionsResult = await collectWebflowList(`${BASE}/sites/${encodeURIComponent(siteId)}/collections`, 'collections', context);
+      const collections = collectionsResult.items;
+      allItems.push(...collections.map((item) => ({ ...item, _collection_type: 'collections' })));
+      errors.push(...collectionsResult.errors); limitations.push(...collectionsResult.limitations);
+      pagesRequested += collectionsResult.pagesRequested; pagesRetrieved += collectionsResult.pagesRetrieved;
+      continuationState ||= collectionsResult.continuationState;
+      connectorError ||= collectionsResult.connectorError;
+      if (collectionsResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (collectionsResult.providerTotal === null) providerTotalKnown = false; else providerTotal += collectionsResult.providerTotal;
 
       if (metricNames.has('collections')) {
         const colMetric = metrics.find((m) => m.external_name === 'collections');
@@ -92,8 +207,15 @@ export const webflowConnector = {
       if (metricNames.has('items')) {
         let totalItems = 0;
         for (const col of collections) {
-          const itemsResult = await providerRequest(`${BASE}/collections/${encodeURIComponent(col.id)}/items?limit=100`, context);
-          totalItems += (itemsResult.items || []).length;
+          const itemsResult = await collectWebflowList(`${BASE}/collections/${encodeURIComponent(col.id)}/items`, 'items', context);
+          totalItems += itemsResult.items.length;
+          allItems.push(...itemsResult.items.map((item) => ({ ...item, _collection_type: 'items', _collection_id: col.id })));
+          errors.push(...itemsResult.errors); limitations.push(...itemsResult.limitations);
+          pagesRequested += itemsResult.pagesRequested; pagesRetrieved += itemsResult.pagesRetrieved;
+          continuationState ||= itemsResult.continuationState;
+          connectorError ||= itemsResult.connectorError;
+          if (itemsResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+          if (itemsResult.providerTotal === null) providerTotalKnown = false; else providerTotal += itemsResult.providerTotal;
         }
         const itemMetric = metrics.find((m) => m.external_name === 'items');
         rows.push({
@@ -105,13 +227,21 @@ export const webflowConnector = {
       }
     }
 
+    const collection = collectionResult({
+      items: allItems,
+      paginationState: { pages_requested: pagesRequested, pages_retrieved: pagesRetrieved, boundary: { max_pages: paginationBoundary(context) } },
+      providerReportedTotal: providerTotalKnown ? providerTotal : null,
+      continuationState,
+      providerCompleteness: providerTotalKnown ? 'declared' : 'unknown',
+      limitations: [...new Set([...limitations, 'Webflow API enforces rate limits of 60 requests per minute.', 'Staged CMS changes require site publish to appear on live domains.'])],
+      errors,
+    });
     return {
       rows,
       cursor: endDate,
-      limitations: [
-        'Webflow API enforces rate limits of 60 requests per minute.',
-        'Staged CMS changes require site publish to appear on live domains.',
-      ],
+      limitations: collection.limitations,
+      collection,
+      ...(connectorError ? { connectorError } : {}),
     };
   },