agentlinkops 0.6.11 → 0.6.16

package/src/verifier/render-evidence.js

@@ -8,7 +8,7 @@ const url = value => text(value, 2048) && validatePublicUrl(value).valid;
 // ready flag cannot turn a challenge, failed session or partial document into proof.
 export function nativeRenderEvidence(doc, version) {
   const invalid = reason => ({ reason });
-  if (version !== 1 || doc.rendererVersion !== 'cf-native-1'
+  if (version !== 1 || !['cf-native-1','alo-native-1'].includes(doc.rendererVersion)
     || !text(doc.contentType) || !text(doc.cfMitigated)
     || typeof doc.challenge !== 'boolean' || typeof doc.visibleLogin !== 'boolean'
     || typeof doc.browserClosed !== 'boolean'
@@ -18,6 +18,9 @@ export function nativeRenderEvidence(doc, version) {
     || !(doc.terminationReason === null || text(doc.terminationReason, 80))
     || !Array.isArray(doc.robotsHistory) || !doc.robotsHistory.length || doc.robotsHistory.length > 150) return invalid('invalid_native_render_evidence');
   if (doc.browserClosed !== true) return invalid('render_close_unconfirmed');
+  if(doc.rendererVersion==='alo-native-1'&&(!doc.cleanup||doc.cleanup.unitStopped!==true||doc.cleanup.cgroupEmpty!==true
+    ||doc.cleanup.namespaceDestroyed!==true||doc.executor?.browserEgressPolicy!=='private-network-unix-bridge-v1'
+    ||doc.executor?.browserEgressEnforced!==true||doc.executor?.version!=='alo-public-node-v1'))return invalid('render_close_unconfirmed');
   if (doc.terminationReason) return invalid('render_acquisition_incomplete');
   if (doc.cfMitigated.toLowerCase() === 'challenge' || doc.challenge) return invalid('render_access_challenge');
   if (doc.cfMitigated) return invalid('render_access_mitigated');
@@ -29,9 +32,25 @@ export function nativeRenderEvidence(doc, version) {
       || row.productToken !== 'LinktrailBot' || !text(row.reason, 80)
       || !Number.isInteger(row.httpStatus) || row.httpStatus < 100 || row.httpStatus > 599
       || (row.crawlDelaySeconds !== null && (!Number.isFinite(row.crawlDelaySeconds) || row.crawlDelaySeconds < 0))) return invalid('invalid_render_robots_evidence');
+    let googlebot;
+    if (row.googlebot !== undefined) {
+      const google = row.googlebot;
+      if (!google || typeof google !== 'object' || Array.isArray(google)
+        || google.productToken !== 'Googlebot'
+        || ![true, false, null].includes(google.allowed)
+        || !(google.reason === null || typeof google.reason === 'string' && /^[a-z0-9_:.-]{1,100}$/u.test(google.reason))
+        || !(google.matchedRule === null || text(google.matchedRule, 8192))
+        || !(google.crawlDelaySeconds === null || Number.isFinite(google.crawlDelaySeconds) && google.crawlDelaySeconds >= 0)) {
+        return invalid('invalid_render_robots_evidence');
+      }
+      // Google permission is evidence, not authority for our browser. The
+      // Linktrail decision above must still independently permit this fetch.
+      googlebot = { productToken: 'Googlebot', allowed: google.allowed, reason: google.reason,
+        matchedRule: google.matchedRule, crawlDelaySeconds: google.crawlDelaySeconds };
+    }
     robotsHistory.push({ sourceUrl: row.sourceUrl, robotsUrl: row.robotsUrl, allowed: true,
       productToken: 'LinktrailBot', reason: row.reason, httpStatus: row.httpStatus,
-      crawlDelaySeconds: row.crawlDelaySeconds });
+      crawlDelaySeconds: row.crawlDelaySeconds, ...(googlebot ? { googlebot } : {}) });
   }
   // Each main-document hop must have a recorded authorization, including the final URL.
   const authorized = new Set(robotsHistory.map(row => row.sourceUrl));
@@ -40,6 +59,10 @@ export function nativeRenderEvidence(doc, version) {
     cfMitigated: doc.cfMitigated, challenge: false, visibleLogin: false, browserClosed: true,
     browserVersion: doc.browserVersion, userAgent: doc.userAgent, requestCount: doc.requestCount,
     durationMs: doc.durationMs, terminationReason: null, robotsHistory };
+  if(doc.rendererVersion==='alo-native-1'){
+    evidence.cleanup={unitStopped:true,cgroupEmpty:true,namespaceDestroyed:true};
+    evidence.executor={version:doc.executor.version,browserEgressEnforced:true,browserEgressPolicy:'private-network-unix-bridge-v1'};
+  }
   for (const key of ['blockedMutationRequests','blockedEssentialRequests']) {
     if (doc[key] === undefined) continue;
     if (!Number.isInteger(doc[key]) || doc[key] < 0 || doc[key] > 150) return invalid('invalid_native_render_evidence');

package/src/verifier/target.js

@@ -0,0 +1,201 @@
+import { validatePublicUrl } from './url.js';
+import { parseDocument, pageDirectives } from './parser.js';
+import { publicFetcher, VerificationError } from './fetch.js';
+import {capturedDocument} from '../page-acquisition/capture.js';
+import { analyzeReadiness, isAccessChallenge } from './readiness.js';
+
+export const TARGET_CHECKER_VERSION = '1';
+export function unknownTargetObservation({targetUrl},{now,reason='timeout'}={}){
+  return {state:'unknown',reason,targetUrl,finalUrl:targetUrl,httpStatus:null,checkedAt:new Date(typeof now==='function'?now():now??Date.now()).toISOString(),
+    redirects:[],target_checker_version:TARGET_CHECKER_VERSION,evidence:{method:'http_html',checkerVersion:TARGET_CHECKER_VERSION,complete:false,fetchedAt:null}};
+}
+const CONFIRMATION_MS = 30 * 60 * 1000;
+const timeOf = (value) => {
+  if (value === null || value === undefined || value === '') return null;
+  const time = new Date(value).getTime();
+  return Number.isFinite(time) ? time : null;
+};
+const bounded = (value, fallback, min, max) => Number.isFinite(value)
+  ? Math.max(min, Math.min(Math.floor(value), max)) : fallback;
+
+async function sha256(data) {
+  return [...new Uint8Array(await crypto.subtle.digest('SHA-256', data))]
+    .map((byte) => byte.toString(16).padStart(2, '0')).join('');
+}
+
+/**
+ * Checks only the supplied destination URL (plus robots and validated redirects).
+ * Healthy means a complete accessible static HTML response, not Google indexing,
+ * semantic page correctness, rendered availability, or any backlink's presence.
+ * publicFetchSafe must assert a transport-level public-only destination boundary.
+ */
+export async function verifyTarget({ targetUrl }, options = {}) {
+  const checkedAt = new Date(typeof options.now === 'function' ? options.now() : options.now ?? Date.now()).toISOString();
+  const result = unknownTargetObservation({targetUrl},{now:checkedAt,reason:'not_checked'});
+  const target = validatePublicUrl(targetUrl);
+  if (!target.valid) { result.reason = `invalid_target:${target.reason}`; return result; }
+  result.targetUrl = result.finalUrl = target.url;
+  if (options.publicFetchSafe !== true) { result.reason = 'public_fetch_not_configured'; return result; }
+
+  if(options.signal?.aborted||(Number.isFinite(options.deadlineAt)&&options.deadlineAt<=Date.now())){result.reason='timeout';return result;}
+  const controller = new AbortController();
+  const ownerAbort=()=>controller.abort();
+  options.signal?.addEventListener('abort',ownerAbort,{once:true});
+  const timer = setTimeout(() => controller.abort(), Math.min(bounded(options.timeoutMs, 20_000, 1, 20_000),options.deadlineAt?Math.max(1,options.deadlineAt-Date.now()):20_000));
+  const fetchOptions = {
+    ...options,
+    robotsPolicy: 'respect',
+    maxBytes: bounded(options.maxBytes, 2 * 1024 * 1024, 1, 2 * 1024 * 1024),
+    maxRedirects: bounded(options.maxRedirects, 5, 0, 5),
+  };
+  try {
+    const { response, body } = await publicFetcher(fetchOptions, controller.signal).source(target.url, result);
+    if(controller.signal.aborted||(Number.isFinite(options.deadlineAt)&&Date.now()>=options.deadlineAt))throw new VerificationError('timeout');
+    result.evidence.fetchedAt = checkedAt;
+    result.evidence.contentType = response.headers.get('content-type');
+    if ([404, 410].includes(response.status)) {
+      result.state = 'unavailable';
+      result.reason = `target_http_${response.status}`;
+      result.evidence.method = 'http_status';
+      result.evidence.complete = true;
+      return result;
+    }
+    result.evidence.bytes = body.bytes;
+    result.evidence.sha256 = await sha256(body.data);
+    result.evidence.decodedSha256 = await sha256(new TextEncoder().encode(body.text));
+    result.evidence.capture=capturedDocument(result,{requestedUrl:target.url,transportId:options.captureTransport?.id??'public_http',
+      transportVersion:options.captureTransport?.version??'1',qualified:options.publicFetchSafe===true,
+      charset:/charset\s*=\s*["']?([^\s;"']+)/iu.exec(response.headers.get('content-type')??'')?.[1]??'utf-8'});
+    result.evidence.etag = response.headers.get('etag');
+    result.evidence.lastModified = response.headers.get('last-modified');
+    if (!body.text.trim()) throw new VerificationError('empty_document');
+    let parsed;
+    try { parsed = parseDocument(body.text, result.finalUrl); }
+    catch (error) { throw new VerificationError(error.reason ?? 'html_parse_error'); }
+    if (parsed.incomplete) throw new VerificationError('incomplete_html', { parseErrors: parsed.parseErrors });
+    const { readiness, managedClientChallenge } = analyzeReadiness({ html: body.text, parsed, targetUrl: target.url });
+    if (isAccessChallenge(parsed, { managedClientChallenge })) throw new VerificationError('access_challenge');
+    if (parsed.meta.some((item) => item.name === 'refresh')) throw new VerificationError('navigation_requires_browser');
+    if (readiness.possibleLoginWall) throw new VerificationError('possible_login_wall');
+    if (parsed.hasScript && !parsed.links.length && parsed.visibleText.length < 80) throw new VerificationError('possible_render_required');
+    if(readiness.requiresRender)throw new VerificationError('possible_render_required');
+    const robotsHeader = response.headers.get('x-robots-tag');
+    if (robotsHeader?.length > 4096) throw new VerificationError('page_directive_too_long');
+    result.directives = pageDirectives(parsed.meta, robotsHeader);
+    result.state = 'healthy';
+    result.reason = 'target_html_available';
+    result.evidence.complete = true;
+    result.evidence.rendered = false;
+    result.evidence.baseUrl = parsed.baseUrl;
+    if (options.includeHtml !== false) result.evidence.html = body.text;
+    if(controller.signal.aborted||(Number.isFinite(options.deadlineAt)&&Date.now()>=options.deadlineAt))throw new VerificationError('timeout');
+    result.warnings = ['Static HTML availability only; rendered content, soft 404s and Google indexing were not verified.'];
+    return result;
+  } catch (error) {
+    result.state = 'unknown';
+    result.evidence.complete = false;
+    const reason = error instanceof VerificationError ? error.reason : controller.signal.aborted ? 'timeout' : 'verification_error';
+    result.reason = reason.replace(/^source_http_/u, 'target_http_');
+    // Carried so a scheduler can come back when the publisher said to, rather than on our cadence.
+    if (Number.isFinite(error?.retryAfterSeconds)) result.retryAfterSeconds = error.retryAfterSeconds;
+    if (result.httpStatus !== null) result.evidence.fetchedAt = checkedAt;
+    if (error.parseErrors) result.evidence.parseErrors = error.parseErrors;
+    return result;
+  } finally {
+    clearTimeout(timer);
+    controller.abort();
+    options.signal?.removeEventListener('abort',ownerAbort);
+  }
+}
+
+function summary(observation, checkedAt = observation?.checkedAt) {
+  if (!observation) return null;
+  const evidence = observation.evidence;
+  return {
+    observationId: observation.observationId ?? observation.id ?? null,
+    state: observation.state, reason: observation.reason,
+    targetUrl: observation.targetUrl, finalUrl: observation.finalUrl,
+    httpStatus: observation.httpStatus, checkedAt,
+    target_checker_version: observation.target_checker_version,
+    // Compact receipts only: full bodies and headers belong to the private evidence object.
+    evidence: evidence ? {
+      complete: evidence.complete, method: evidence.method, checkerVersion: evidence.checkerVersion,
+      sha256: evidence.sha256, decodedSha256:evidence.decodedSha256, bytes: evidence.bytes, fetchedAt: evidence.fetchedAt,
+    } : undefined,
+  };
+}
+
+/** Pure reducer. Persist atomically with the target observation and target event. */
+export function transitionTargetState(previous, observation, now = Date.now()) {
+  const prior = previous ?? {};
+  const checkedMs = timeOf(observation.checkedAt) ?? timeOf(typeof now === 'function' ? now() : now);
+  if (checkedMs === null) throw new TypeError('A valid observation timestamp is required');
+  const checkedAt = new Date(checkedMs).toISOString();
+  const base = {
+    state: prior.state ?? 'unknown',
+    confirmedState: prior.confirmedState ?? (['healthy', 'unavailable'].includes(prior.state) ? prior.state : null),
+    latestAttempt: prior.latestAttempt ?? null,
+    lastSuccessfulObservation: prior.lastSuccessfulObservation ?? null,
+    lastSuccessfulAt: prior.lastSuccessfulAt ?? null,
+    lastHealthyAt: prior.lastHealthyAt ?? null,
+    firstUnavailableAt: prior.firstUnavailableAt ?? null,
+    unavailableCount: prior.unavailableCount ?? 0,
+    wasEverHealthy: prior.wasEverHealthy ?? prior.lastSuccessfulObservation?.state === 'healthy',
+    uncertain: prior.uncertain ?? true,
+    // Carried forward so a later conclusive observation does not silently drop a floor the
+    // publisher set and the scheduler has not yet reached.
+    retryNotBefore: prior.retryNotBefore ?? null,
+    nextCheckAt: null, event: null, ignored: false,
+  };
+  const latestMs = timeOf(prior.latestAttempt?.checkedAt);
+  const observationId = observation.observationId ?? observation.id;
+  const repeatedId = observationId && [prior.latestAttempt?.observationId, prior.lastSuccessfulObservation?.observationId].includes(observationId);
+  if ((latestMs !== null && checkedMs <= latestMs) || repeatedId) {
+    return { ...base, nextCheckAt: prior.nextCheckAt ?? null, ignored: true };
+  }
+  base.latestAttempt = summary(observation, checkedAt);
+  if (!['healthy', 'unavailable', 'unknown'].includes(observation.state)) throw new TypeError('Invalid target observation state');
+  const conclusive = observation.evidence?.complete === true
+    && ((observation.state === 'healthy' && observation.httpStatus === 200)
+      || (observation.state === 'unavailable' && [404, 410].includes(observation.httpStatus)));
+  if (!conclusive) {
+    base.uncertain = true;
+    // A publisher that answered 429 or 503 may have said when to come back. It is a FLOOR, not a
+    // schedule: "not before" is not "then". So it is surfaced as its own field and the scheduler
+    // takes the later of this and the target's own cadence — writing it as the next check time
+    // would re-check a rate-limiting host every ninety seconds, which is worse than ignoring it.
+    if (Number.isFinite(observation.retryAfterSeconds)) {
+      base.retryNotBefore = new Date(checkedMs + observation.retryAfterSeconds * 1000).toISOString();
+    }
+    return base;
+  }
+
+  base.lastSuccessfulObservation = base.latestAttempt;
+  base.lastSuccessfulAt = checkedAt;
+  base.uncertain = false;
+  if (observation.state === 'healthy') {
+    base.state = base.confirmedState = 'healthy';
+    base.lastHealthyAt = checkedAt;
+    base.firstUnavailableAt = null;
+    base.unavailableCount = 0;
+    base.wasEverHealthy = true;
+    const type = prior.confirmedState === 'unavailable' || prior.state === 'unavailable'
+      ? 'target_recovered' : !prior.wasEverHealthy && !prior.lastHealthyAt ? 'target_healthy' : null;
+    if (type) base.event = { type, before: summary(prior.lastSuccessfulObservation), after: base.latestAttempt };
+    return base;
+  }
+
+  const first = timeOf(base.firstUnavailableAt);
+  base.firstUnavailableAt = first === null ? checkedAt : base.firstUnavailableAt;
+  base.unavailableCount = first === null ? 1 : base.unavailableCount + 1;
+  const confirmed = first !== null && checkedMs - first >= CONFIRMATION_MS && base.unavailableCount >= 2;
+  base.state = confirmed ? 'unavailable' : 'suspected_unavailable';
+  if (!confirmed) base.nextCheckAt = new Date((first ?? checkedMs) + CONFIRMATION_MS).toISOString();
+  else {
+    base.confirmedState = 'unavailable';
+    if (prior.confirmedState !== 'unavailable' && prior.state !== 'unavailable') {
+      base.event = { type: 'target_unavailable', before: summary(prior.lastSuccessfulObservation), after: base.latestAttempt };
+    }
+  }
+  return base;
+}

package/src/verifier/transition.js

@@ -1,4 +1,5 @@
 const CONFIRMATION_MS = 30 * 60 * 1000;
+import {recoverySummary} from '../page-acquisition/recovery.js';
 
 const timeOf = (value) => {
   if (value === null || value === undefined || value === '') return null;
@@ -6,9 +7,13 @@ const timeOf = (value) => {
   return Number.isFinite(time) ? time : null;
 };
 
+// Retain a compact access history without private transport receipts or captures.
+const acquisitionSummary=recoverySummary;
+
 function summary(observation) {
   if (!observation) return null;
   const { evidence } = observation;
+  const acquisition = acquisitionSummary(evidence?.acquisition);
   const all = observation.occurrences ?? [];
   // State/event snapshots are small references to full observations in D1/R2.
   // linkSignature covers ALL occurrences, preserving change detection beyond excerpts.
@@ -31,7 +36,8 @@ function summary(observation) {
     directives: observation.directives ? { noindex: observation.directives.noindex, nofollow: observation.directives.nofollow, indexingStatus: observation.directives.indexingStatus } : undefined,
     evidence: evidence ? {
       complete: evidence.complete, method: evidence.method, checkerVersion: evidence.checkerVersion,
-      sha256: evidence.sha256, bytes: evidence.bytes, fetchedAt: evidence.fetchedAt,
+      sha256: evidence.sha256, decodedSha256: evidence.decodedSha256, bytes: evidence.bytes, fetchedAt: evidence.fetchedAt,
+      ...(acquisition?{acquisition}:{}),
     } : undefined,
   };
 }
@@ -49,12 +55,17 @@ function fingerprint(observation) {
 /** Pure state reducer. Persist its result atomically with the observation + outbox. */
 export function transitionState(previous, observation, now = Date.now()) {
   const prior = previous ?? {};
+  // Keep a historical confirmed fact separate from the latest check's ability
+  // to verify it. Older saved states predate this explicit historical field.
+  const confirmedState = prior.confirmedState ?? (prior.state === 'present' || prior.state === 'confirmed_missing'
+    || prior.state === 'source_unavailable' && prior.sourceUnavailableConfirmed === true ? prior.state : null);
   const checkedMs = timeOf(observation.checkedAt) ?? timeOf(typeof now === 'function' ? now() : now);
   if (checkedMs === null) throw new TypeError('A valid observation timestamp is required');
   const checkedAt = new Date(checkedMs).toISOString();
   const latestMs = prior.latestAttempt ? timeOf(prior.latestAttempt.checkedAt) : null;
   const base = {
     state: prior.state ?? 'unknown',
+    confirmedState,
     latestAttempt: prior.latestAttempt ?? null,
     lastSuccessfulObservation: prior.lastSuccessfulObservation ?? null,
     firstAbsentAt: prior.firstAbsentAt ?? null,
@@ -72,6 +83,7 @@ export function transitionState(previous, observation, now = Date.now()) {
   if (latestMs !== null && checkedMs <= latestMs) return { ...base, nextCheckAt: prior.nextCheckAt ?? null, ignored: true };
   base.latestAttempt = summary({ ...observation, checkedAt });
   if (observation.state === 'unknown' || observation.evidence?.complete !== true) {
+    base.state = 'unknown';
     base.uncertain = true;
     // A scheduler decides retry/backoff after an unknown; no past due timestamp loop. The one
     // thing carried is a floor the publisher stated in a Retry-After — "not before", never "then",
@@ -86,6 +98,7 @@ export function transitionState(previous, observation, now = Date.now()) {
   base.uncertain = false;
   if (observation.state === 'present') {
     base.state = 'present';
+    base.confirmedState = 'present';
     base.wasEverPresent = true;
     base.firstAbsentAt = null;
     base.absentCount = 0;
@@ -94,7 +107,7 @@ export function transitionState(previous, observation, now = Date.now()) {
     base.sourceUnavailableConfirmed = false;
     let type;
     if (!prior.wasEverPresent && prior.lastSuccessfulObservation?.state !== 'present') type = 'placement_acquired';
-    else if (prior.state === 'confirmed_missing' || prior.sourceUnavailableConfirmed) type = 'placement_recovered';
+    else if (confirmedState === 'confirmed_missing' || prior.sourceUnavailableConfirmed) type = 'placement_recovered';
     else if (prior.lastSuccessfulObservation?.state === 'present' && fingerprint(prior.lastSuccessfulObservation) !== fingerprint(observation)) type = 'placement_changed';
     if (type) base.event = { type, before: summary(prior.lastSuccessfulObservation), after: base.latestAttempt };
     return base;
@@ -108,8 +121,9 @@ export function transitionState(previous, observation, now = Date.now()) {
     base.sourceUnavailableConfirmed = false;
     const confirmed = first !== null && checkedMs - first >= CONFIRMATION_MS && base.absentCount >= 2;
     base.state = confirmed ? 'confirmed_missing' : 'suspected_missing';
+    if (confirmed) base.confirmedState = 'confirmed_missing';
     if (!confirmed) base.nextCheckAt = new Date((first ?? checkedMs) + CONFIRMATION_MS).toISOString();
-    if (confirmed && prior.state !== 'confirmed_missing' && base.wasEverPresent) {
+    if (confirmed && confirmedState !== 'confirmed_missing' && base.wasEverPresent) {
       base.event = { type: 'placement_lost', before: summary(prior.lastSuccessfulObservation), after: base.latestAttempt };
     }
     return base;
@@ -121,6 +135,7 @@ export function transitionState(previous, observation, now = Date.now()) {
   base.absentCount = 0;
   base.state = 'source_unavailable';
   base.sourceUnavailableConfirmed = first !== null && checkedMs - first >= CONFIRMATION_MS && base.unavailableCount >= 2;
+  if (base.sourceUnavailableConfirmed) base.confirmedState = 'source_unavailable';
   if (!base.sourceUnavailableConfirmed) base.nextCheckAt = new Date((first ?? checkedMs) + CONFIRMATION_MS).toISOString();
   if (base.sourceUnavailableConfirmed && !prior.sourceUnavailableConfirmed) {
     base.event = { type: 'source_unavailable', before: summary(prior.lastSuccessfulObservation), after: base.latestAttempt };

package/src/verifier/url.js

@@ -54,3 +54,34 @@ export function matchesTarget(candidate, target, scope) {
       return false;
   }
 }
+
+const REDIRECT_URL_PARAMETERS = new Set(['url', 'redirect_url', 'redirect_uri', 'destination', 'target']);
+const MAX_WRAPPER_PARAMETERS = 4096;
+
+/**
+ * A physical anchor may declare a destination through a redirect service.
+ * This is only a reason to withhold absence, never a matching occurrence or
+ * permission to request that URL. Return no publisher URL or token-bearing data.
+ */
+export function hasRedirectWrapperToTarget(links, targetUrl, targetScope = 'exact') {
+  if (!Array.isArray(links) || !['exact', 'domain', 'subdomain', 'path'].includes(targetScope)) return false;
+  const target = validatePublicUrl(targetUrl);
+  if (!target.valid) return false;
+  for (const link of links) {
+    if (typeof link?.targetUrl !== 'string' || !link.targetUrl.includes('?')) continue;
+    const wrapper = validatePublicUrl(link.targetUrl);
+    if (!wrapper.valid || wrapper.url.length > 8192) continue;
+    let parameters = 0;
+    // URLSearchParams decodes the query once. No recursive URL traversal,
+    // repeated decoding, substring search or hydration/text interpretation.
+    // The URL's 8192-character bound cannot contain over 4096 nonempty query
+    // entries, even with one-character names and no values; no valid tail is cut.
+    for (const [name, value] of new URL(wrapper.url).searchParams) {
+      if (++parameters > MAX_WRAPPER_PARAMETERS) break;
+      if (!REDIRECT_URL_PARAMETERS.has(name)) continue;
+      const destination = validatePublicUrl(value);
+      if (destination.valid && matchesTarget(destination.url, target.url, targetScope)) return true;
+    }
+  }
+  return false;
+}

package/src/watch-crawlability.js

@@ -0,0 +1,144 @@
+import { pageDirectives } from './page-extraction/html.js';
+import { validateCaptureReceipt } from './page-acquisition/contracts.js';
+import { nativeRenderEvidence } from './verifier/render-evidence.js';
+
+const reason = value => typeof value === 'string' && /^[a-z0-9_:.-]{1,100}$/u.test(value) ? value : null;
+const date = value => typeof value === 'string' && Number.isFinite(Date.parse(value)) ? value : null;
+const bool = value => typeof value === 'boolean' ? value : null;
+
+function sourceObservation(watch, observation) {
+  const result = observation?.result;
+  const observedAt = date(observation?.checked_at ?? result?.checkedAt);
+  if (!result || result.sourceUrl !== watch.source_url || !observedAt || typeof observation.id !== 'string') return null;
+  return { result, observedAt, id: observation.id };
+}
+function completePage(observation) {
+  const result = observation?.result;
+  // The verifier initializes empty directives before fetching. A 404, failed parse,
+  // access wall or incomplete capture must never turn those defaults into facts.
+  if (result?.evidence?.complete === true && ['present', 'absent'].includes(result.state)) return true;
+  // This verdict is reached only after a complete HTML body, successful parsing,
+  // directives and the access/login-wall checks. Rendering can still be required
+  // to verify the link; it does not erase already observed page directives.
+  const evidence = result?.evidence;
+  const parsedShell = result?.reason === 'possible_render_required'
+    && evidence?.readiness?.possibleLoginWall === false
+    && typeof evidence?.readiness?.requiresRender === 'boolean';
+  const staticShell = parsedShell && evidence?.method === 'http_html';
+  const unconfirmedDom = (parsedShell || ['render_no_positive_evidence', 'render_readiness_unconfirmed'].includes(result?.reason))
+    && evidence?.method === 'browser_html' && evidence.rendered === true && typeof evidence.ready === 'boolean';
+  if (result?.state !== 'unknown' || !(staticShell || unconfirmedDom)
+    || result.httpStatus !== 200
+    || !/^text\/html(?:\s*;|$)/iu.test(evidence.contentType ?? '')
+    || !Array.isArray(result.directives?.meta)
+    || !result.directives.meta.every(item => typeof item?.name === 'string' && typeof item?.content === 'string')
+    || !Array.isArray(result.directives?.xRobotsTag)
+    || !result.directives.xRobotsTag.every(item => typeof item === 'string')) return false;
+  try {
+    const capture = validateCaptureReceipt(evidence.capture, { finalUrl: result.finalUrl });
+    if (unconfirmedDom && (!Array.isArray(result.redirects) || nativeRenderEvidence({ ...evidence.browser,
+      finalUrl: result.finalUrl, redirects: result.redirects }, 1).reason)) return false;
+    return capture.representation === (staticShell ? 'raw_html' : 'rendered_dom') && capture.requestedUrl === result.sourceUrl
+      && capture.httpStatus === 200 && capture.body.bytes > 0
+      && capture.body.bytes === evidence.bytes && capture.body.sha256 === evidence.sha256
+      && capture.body.decodedSha256 === evidence.decodedSha256
+      && capture.acquiredAt === evidence.fetchedAt;
+  } catch { return false; }
+}
+function completeLink(watch, observation) {
+  const result = observation?.result;
+  return result?.evidence?.complete === true && ['present', 'absent'].includes(result.state)
+    && result.targetUrl === watch.target_url
+    && result.targetScope === watch.target_scope;
+}
+function googleDirectives(directives) {
+  if (!directives || typeof directives !== 'object') return { noindex: null, nofollow: null };
+  if (directives.googlebot) return {
+    noindex: bool(directives.googlebot.noindex), nofollow: bool(directives.googlebot.nofollow),
+  };
+  // Old full observations retain the actual meta/header inputs. Reuse the same
+  // targeted-directive parser instead of inferring Google facts from our bot.
+  if (Array.isArray(directives.meta) && directives.meta.every(item => typeof item?.name === 'string' && typeof item?.content === 'string')
+    && Array.isArray(directives.xRobotsTag) && directives.xRobotsTag.every(item => typeof item === 'string')) {
+    return pageDirectives(directives.meta, directives.xRobotsTag.join(',')).googlebot;
+  }
+  return { noindex: directives.noindex === true ? true : null,
+    nofollow: directives.nofollow === true ? true : null };
+}
+function observedRel(observation) {
+  const result = observation?.result;
+  if (result?.state !== 'present' || result.occurrencesTruncated === true
+    || !Array.isArray(result.occurrences) || !result.occurrences.length) return null;
+  const tokens = new Set();
+  for (const occurrence of result.occurrences) {
+    if (!Array.isArray(occurrence?.rel)) return null;
+    for (const value of occurrence.rel) {
+      if (typeof value !== 'string' || !/^[a-z0-9_-]{1,40}$/iu.test(value)) return null;
+      tokens.add(value.toLowerCase());
+    }
+  }
+  return [...tokens].sort();
+}
+function latestRobots(observation) {
+  const result = observation?.result;
+  const native = result?.evidence?.browser?.robotsHistory;
+  if (Array.isArray(native) && native.length) {
+    const urls = new Set([result.sourceUrl, result.finalUrl].filter(value => typeof value === 'string'));
+    for (const hop of Array.isArray(result.redirects) ? result.redirects : []) {
+      if (typeof hop?.from === 'string') urls.add(hop.from);
+      if (typeof hop?.to === 'string') urls.add(hop.to);
+    }
+    // Native traces also contain script/font requests. Only the document's
+    // actual URL chain supports a claim about Google crawling this page.
+    const rows = native.filter(row => urls.has(row?.sourceUrl));
+    for (const url of urls) if (!rows.some(row => row.sourceUrl === url)) rows.push(null);
+    return { rows, observedAt: observation.observedAt };
+  }
+  if (Array.isArray(result?.robotsHistory) && result.robotsHistory.length) return { rows: result.robotsHistory, observedAt: observation.observedAt };
+  if (result?.robots) return { rows: [result.robots], observedAt: observation.observedAt };
+  const original = result?.evidence?.staticObservation;
+  return { rows: Array.isArray(original?.robotsHistory) ? original.robotsHistory : [],
+    observedAt: date(original?.checkedAt) ?? observation?.observedAt ?? null };
+}
+
+/** Read-only facts from saved observations; never a live Google indexing test. */
+export function projectWatchCrawlability(watch, { latestObservation = null, latestPageObservation = null, lastSuccessfulObservation = null } = {}) {
+  const latest = sourceObservation(watch, latestObservation);
+  const savedPage = sourceObservation(watch, latestPageObservation);
+  const successful = sourceObservation(watch, lastSuccessfulObservation);
+  const page = [latest, savedPage, successful].filter(completePage)
+    .sort((a, b) => Date.parse(b.observedAt) - Date.parse(a.observedAt))[0] ?? null;
+  const verified = completeLink(watch, latest) ? latest : completeLink(watch, successful) ? successful : null;
+  const directives = googleDirectives(page?.result.directives);
+  const currentVerifiable = completeLink(watch, latest);
+  const projection = {
+    google_crawlable: null,
+    google_crawl_reason: latest ? 'googlebot_not_observed' : 'not_checked',
+    robots_rule: null,
+    noindex: directives.noindex,
+    page_nofollow: directives.nofollow,
+    link_rel: verified ? observedRel(verified) : null,
+    can_verify_reason: currentVerifiable ? null : reason(latest?.result.reason)
+      ?? (latest ? 'verification_unknown' : latestObservation ? 'observation_source_mismatch' : 'not_checked'),
+    last_verified_at: verified?.observedAt ?? null,
+    crawlability_provenance: {
+      robots_observed_at: null, page_observed_at: page?.observedAt ?? null,
+      link_observed_at: verified?.result.state === 'present' ? verified.observedAt : null,
+      robots_observation_id: null, page_observation_id: page?.id ?? null,
+      link_observation_id: verified?.result.state === 'present' ? verified.id : null,
+    },
+  };
+  const { rows, observedAt } = latestRobots(latest);
+  if (!rows.length) return projection;
+  const decisions = rows.map(row => row?.googlebot?.productToken === 'Googlebot' ? row.googlebot : null);
+  const blocked = decisions.find(item => item?.allowed === false);
+  const unknown = decisions.find(item => !item || bool(item.allowed) === null);
+  const decision = blocked ?? unknown ?? decisions.at(-1);
+  projection.google_crawlable = blocked ? false : unknown !== undefined ? null : true;
+  projection.google_crawl_reason = reason(decision?.reason)
+    ?? (blocked ? 'robots_disallowed' : unknown !== undefined ? 'googlebot_not_observed' : 'robots_allowed');
+  projection.robots_rule = typeof decision?.matchedRule === 'string' ? decision.matchedRule.slice(0, 8192) : null;
+  projection.crawlability_provenance.robots_observed_at = observedAt;
+  projection.crawlability_provenance.robots_observation_id = latest.id;
+  return projection;
+}

package/skills/content-defingerprinting/SKILL.md

@@ -1,29 +0,0 @@
----
-name: content-defingerprinting
-description: Run the required invisible-character and text-artifact cleanup after humanizing public content, using this package's watermark scanner. Preserve citations, source evidence and meaningful text.
----
-
-# Content defingerprinting
-
-This is stage 2b of the content workflow: deterministic text cleanup after the separate
-editorial humanize pass. Read this skill's `scripts/watermarks/README.md` for
-scanner behavior and the distinction between authored prose and archived evidence.
-
-Run against the final authored files, including titles, descriptions, email/social
-copy and page text embedded in templates:
-
-```bash
-python3 scripts/watermarks/wm-scan.py <authored-file>   # relative to this skill's directory
-```
-
-Inspect findings before changing them. For removable artifacts in authored prose,
-use `--fix` and rerun the scanner until it passes. Preserve the scanner's protections
-for archived evidence, sourced quotations, meaningful Unicode and load-bearing code
-literals; resolve those findings individually. Do not rewrite sources to clear a check.
-
-For content composed directly in a browser, first save the final authored text to a
-local UTF-8 draft, scan that file, and paste the checked version. Repeat if text changes.
-This step does not replace the humanizer or alter source/provenance records.
-
-Then use `internal-linking-optimizer`. Record the scanned paths and outcome with the
-content's normal review notes. Do not call an unrun scan passed.