agentlinkops 0.6.1 → 0.6.3

package/src/corpus/acquire.js

@@ -13,8 +13,11 @@ export const FETCH_KINDS = Object.freeze(['direct', 'rendered', 'proxied']);
 export const ACQUISITION_LIMITS = Object.freeze({ maxBytes: 2 * 1024 * 1024, timeoutMs: 20_000, maxRedirects: 5 });
 
 // Every way a page attempt can end. The frontier records the outcome; none of them is ever
-// allowed to reach the corpus as "this page has no links".
-export const OUTCOMES = Object.freeze(['fetched', 'not_found', 'blocked', 'unavailable', 'not_configured', 'deferred']);
+// allowed to reach the corpus as "this page has no links". `unchanged` is the conditional-GET
+// end: a 304 against validators we sent. It is a real observation with provenance, and it is
+// recorded in the refresh ledger (corpus_refresh_checks), never as a corpus_pages row — a page
+// we did not re-read is not a page we fetched, and the pages table has no vocabulary for it.
+export const OUTCOMES = Object.freeze(['fetched', 'not_found', 'blocked', 'unavailable', 'not_configured', 'deferred', 'unchanged']);
 
 const BLOCKED_REASONS = /^(robots_disallowed|robots_|access_challenge|source_http_(401|403|429))/;
 // A slot we could not take is not the publisher's fault and not a defect: retry it later.
@@ -48,7 +51,7 @@ export function createAcquisition(options = {}) {
   const runtimes = options.runtimes ?? {};
   const clock = options.now ?? (() => new Date().toISOString());
 
-  async function direct(url, beforeFetch) {
+  async function direct(url, beforeFetch, { validators = null, document = false } = {}) {
     // The public boundary is a property of the transport, not of this module. Refuse rather
     // than fetch on an unasserted one; a lexically public hostname can still resolve inside.
     if (options.publicFetchSafe !== true) throw new AcquisitionError('public_fetch_boundary_not_asserted');
@@ -57,9 +60,12 @@ export function createAcquisition(options = {}) {
     const trace = { finalUrl: url, redirects: [], robots: null, httpStatus: null };
     try {
       const fetcher = publicFetcher({ ...options, beforeFetch: beforeFetch ?? options.beforeFetch, maxBytes, maxRedirects: ACQUISITION_LIMITS.maxRedirects }, controller.signal);
-      const { response, body } = await fetcher.source(url, trace);
+      const { response, body, notModified, validators: earned } = await fetcher.source(url, trace, { validators, document });
+      if (notModified) return { outcome: 'unchanged', trace, html: null, validators: earned };
       if (NOT_FOUND.has(response.status)) return { outcome: 'not_found', trace, html: null };
-      return { outcome: 'fetched', trace, html: body?.text ?? '', bytes: body?.bytes ?? 0 };
+      return { outcome: 'fetched', trace, html: body?.text ?? '', bytes: body?.bytes ?? 0,
+        linkHeader: response.headers.get('link'),
+        validators: { etag: response.headers.get('etag'), lastModified: response.headers.get('last-modified') } };
     } finally {
       clearTimeout(timer);
     }
@@ -93,20 +99,26 @@ export function createAcquisition(options = {}) {
     /**
      * Acquires one page. Always resolves: an attempt that failed is a recorded outcome, not
      * a thrown error, because the corpus must keep attempted outcomes and not only successes.
+     *
+     * `{ validators }` makes the direct path a conditional GET: pass the ETag/Last-Modified a
+     * previous fetch of this URL earned, and an unchanged page comes back as outcome
+     * `unchanged` with `http_status` 304 and no body — a dated observation, not an absence.
      */
-    async fetchPage(rawUrl, { kind = 'direct', beforeFetch = null } = {}) {
+    async fetchPage(rawUrl, { kind = 'direct', beforeFetch = null, validators = null } = {}) {
       const at = clock();
       const base = { url: rawUrl, final_url: null, kind, http_status: null, robots: null,
-        html: null, body_truncated: false, fetched_at: at, reason: null };
+        html: null, body_truncated: false, fetched_at: at, reason: null, etag: null, last_modified: null };
       if (!FETCH_KINDS.includes(kind)) return { ...base, outcome: 'unavailable', reason: 'unsupported_fetch_kind' };
       const checked = validatePublicUrl(rawUrl);
       if (!checked.valid) return { ...base, outcome: 'unavailable', reason: `unsafe_url:${checked.reason}` };
 
       try {
         if (kind === 'direct') {
-          const { outcome, trace, html } = await direct(checked.url, beforeFetch);
+          const { outcome, trace, html, validators: earned, linkHeader } = await direct(checked.url, beforeFetch, { validators });
           return { ...base, outcome, final_url: trace.finalUrl, http_status: trace.httpStatus,
             robots: trace.robots, redirects: trace.redirects, html,
+            etag: earned?.etag ?? null, last_modified: earned?.lastModified ?? null,
+            link_header: linkHeader ?? null,
             // The verifier's reader refuses an oversized body outright, so a returned body was
             // read to the end. Truncation reaches extraction from runtimes with their own caps.
             body_truncated: false, render_ms: 0, proxy_bytes: 0 };
@@ -131,5 +143,32 @@ export function createAcquisition(options = {}) {
         return { ...base, outcome, reason };
       }
     },
+
+    /**
+     * Acquires one non-page document: a sitemap or an RSS/Atom feed. Same transport, same
+     * robots-per-hop, same public boundary and same outcome vocabulary as `fetchPage`; only
+     * the Accept header and the content-type gate differ, because the XML family is the
+     * expected representation here. Conditional GET works the same way — feeds revalidate too.
+     *
+     * The crawl loop never calls this; it exists for the freshness lanes (sitemap diffing,
+     * feed polling) that read publisher declarations rather than pages.
+     */
+    async fetchDocument(rawUrl, { beforeFetch = null, validators = null } = {}) {
+      const at = clock();
+      const base = { url: rawUrl, final_url: null, kind: 'direct', http_status: null,
+        body: null, fetched_at: at, reason: null, etag: null, last_modified: null };
+      const checked = validatePublicUrl(rawUrl);
+      if (!checked.valid) return { ...base, outcome: 'unavailable', reason: `unsafe_url:${checked.reason}` };
+      try {
+        const { outcome, trace, html, validators: earned, linkHeader } = await direct(checked.url, beforeFetch, { validators, document: true });
+        return { ...base, outcome, final_url: trace.finalUrl, http_status: trace.httpStatus,
+          body: outcome === 'fetched' ? html : null,
+          etag: earned?.etag ?? null, last_modified: earned?.lastModified ?? null,
+          link_header: linkHeader ?? null };
+      } catch (error) {
+        const reason = typeof error?.reason === 'string' ? error.reason : 'acquisition_error';
+        return { ...base, outcome: classifyFailure(reason), reason };
+      }
+    },
   };
 }

package/src/corpus/seeds.js

@@ -18,7 +18,81 @@ export class SeedError extends Error {
   constructor(reason) { super(reason); this.name = 'SeedError'; this.reason = reason; }
 }
 
-function decodeXml(value) {
+// The seed-composition cap (DP-0049-T01 Fix 2). The birding corpus took 32.3% of its pages
+// from six .edu library-guide seed hosts and the citation graph followed the seeds: 59 of 100
+// shortlist rows were library infrastructure. The cap keeps one seed CLASS from dominating a
+// corpus no matter how large its sitemaps are, without touching the per-host cap that keeps
+// one large publisher from doing the same.
+export const SEED_COMPOSITION = Object.freeze({ maxShare: 1 / 3 });
+
+/**
+ * Parses a seeds file into ordered `{ host, class }` records. Every comment line opens a
+ * class for the host lines that follow it, so the class headers the earlier seed files
+ * already carried de facto ("--- surfaced by Q1 ... ---") became the explicit unit the
+ * composition cap enforces. A bare `#` with nothing after it resets to `unclassified`.
+ */
+export function parseSeedHosts(text) {
+  if (typeof text !== 'string') throw new SeedError('seeds_not_text');
+  const hosts = [];
+  let klass = 'unclassified';
+  for (const line of text.split(/\r?\n/u)) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    if (trimmed.startsWith('#')) {
+      const label = trimmed.replace(/^#+\s*/u, '').trim();
+      klass = label || 'unclassified';
+      continue;
+    }
+    hosts.push({ host: trimmed.toLowerCase(), class: klass });
+  }
+  return hosts;
+}
+
+/**
+ * Caps each class's seed-URL contribution. `classHosts` is one record per class in file
+ * order: `{ class, hosts: [[url, ...], ...] }` — the per-host URL lists AFTER the per-host
+ * cap, in admission order. A class over the cap is down-sampled round-robin across its
+ * hosts so the cut lands on every host evenly instead of beheading the last few; the share
+ * is computed against the PRE-CAP total and nothing is redistributed to other classes, so
+ * the rule is deterministic and auditable. A single-class run is never cut: one class cannot
+ * dominate itself.
+ *
+ * @returns The kept URLs in class order, and the per-class accounting the run report carries.
+ */
+export function capSeedComposition(classHosts, { maxShare = SEED_COMPOSITION.maxShare } = {}) {
+  if (!Array.isArray(classHosts) || classHosts.some(entry => !entry || !Array.isArray(entry.hosts))) {
+    throw new SeedError('seed_classes_malformed');
+  }
+  const total = classHosts.reduce((sum, entry) => sum + entry.hosts.reduce((n, urls) => n + urls.length, 0), 0);
+  const cap = Math.max(1, Math.floor(maxShare * total));
+  const multiClass = classHosts.filter(entry => entry.hosts.some(urls => urls.length)).length > 1;
+  const kept = [], perClass = [];
+  for (const entry of classHosts) {
+    const offered = entry.hosts.reduce((n, urls) => n + urls.length, 0);
+    let keptUrls = [];
+    if (multiClass && offered > cap) {
+      // Round-robin one URL per host per turn, hosts in file order, until the cap is hit.
+      const cursors = entry.hosts.map(() => 0);
+      while (keptUrls.length < cap) {
+        let advanced = false;
+        for (let h = 0; h < entry.hosts.length && keptUrls.length < cap; h++) {
+          if (cursors[h] < entry.hosts[h].length) {
+            keptUrls.push(entry.hosts[h][cursors[h]++]);
+            advanced = true;
+          }
+        }
+        if (!advanced) break;
+      }
+    } else {
+      keptUrls = entry.hosts.flat();
+    }
+    kept.push(...keptUrls);
+    perClass.push({ class: entry.class, offered, kept: keptUrls.length, cut: offered - keptUrls.length });
+  }
+  return { kept, cap, applied: multiClass && perClass.some(entry => entry.cut > 0), perClass };
+}
+
+export function decodeXml(value) {
   return value.replace(/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]+));/giu, (whole, dec, hex, name) => {
     if (dec || hex) {
       const code = Number.parseInt(dec ?? hex, dec ? 10 : 16);
@@ -84,7 +158,7 @@ export function withinSitemapScope(candidate, sitemapUrl, scope = 'path') {
  *
  * @throws SeedError when the document is too large or is not a sitemap at all.
  */
-export function parseSitemap(input, { baseUrl, enforceScope = true, scope = 'path' } = {}) {
+export function parseSitemapEntries(input, { baseUrl, enforceScope = true, scope = 'path' } = {}) {
   if (typeof input !== 'string') throw new SeedError('sitemap_not_text');
   if (input.length > SEED_LIMITS.documentBytes) throw new SeedError('sitemap_too_large');
   // A byte-order mark before the root element defeats every subsequent match.
@@ -98,12 +172,24 @@ export function parseSitemap(input, { baseUrl, enforceScope = true, scope = 'pat
   if (indexAt < 0 && setAt < 0) return parseTextSitemap(xml, { baseUrl, enforceScope, scope });
   const kind = indexAt >= 0 && (setAt < 0 || indexAt < setAt) ? 'index' : 'urlset';
 
-  const urls = [];
+  const entries = [];
+  const seen = new Set();
   const skipped = {};
   let truncated = false;
+  // Collected rather than iterated lazily because each entry's lastmod is searched in the
+  // region up to the NEXT `<loc>`, which needs lookahead. Collection stops one past the cap:
+  // a document with more `<loc>` blocks than the cap is knowably truncated without reading
+  // all of them, which is the same signal the lazy scan's early break used to produce.
   const pattern = /<\s*loc\s*>([\s\S]*?)<\s*\/\s*loc\s*>/giu;
+  const matches = [];
   for (const match of xml.matchAll(pattern)) {
-    if (urls.length >= SEED_LIMITS.urlsPerDocument) { truncated = true; break; }
+    matches.push(match);
+    if (matches.length > SEED_LIMITS.urlsPerDocument) break;
+  }
+  const lastmodPattern = /<\s*lastmod\s*>([\s\S]*?)<\s*\/\s*lastmod\s*>/iu;
+  for (let index = 0; index < matches.length; index++) {
+    const match = matches[index];
+    if (entries.length >= SEED_LIMITS.urlsPerDocument) { truncated = true; break; }
     let raw = decodeXml(match[1].replace(/<!\[CDATA\[([\s\S]*?)\]\]>/gu, '$1')).trim();
     if (!raw) continue;
     // The protocol requires an absolute URL here. Resolving a relative value against the
@@ -114,9 +200,27 @@ export function parseSitemap(input, { baseUrl, enforceScope = true, scope = 'pat
       skipped.out_of_sitemap_scope = (skipped.out_of_sitemap_scope ?? 0) + 1;
       continue;
     }
-    if (!urls.includes(checked.url)) urls.push(checked.url);
+    if (seen.has(checked.url)) continue;
+    seen.add(checked.url);
+    // A lastmod belongs to the `<url>` (or `<sitemap>`) block its `<loc>` opens; the region
+    // ends where the next entry begins. Captured RAW: this is the publisher's declared change
+    // time, parsed where it is spent (`parseTimestamp` in freshness.js), and an unparsable
+    // claim is counted there rather than silently dropped here.
+    const region = xml.slice(match.index + match[0].length, matches[index + 1]?.index ?? xml.length);
+    const lastmodRaw = lastmodPattern.exec(region)?.[1]?.trim() ?? null;
+    entries.push({ loc: checked.url, lastmod: lastmodRaw });
   }
-  return { kind, urls, truncated, skipped };
+  // The loop can fill the cap on the LAST collected match, with nothing left to iterate and
+  // trip the in-loop break. Collection stopping at cap+1 is itself the proof that more
+  // `<loc>` blocks existed, which is exactly what truncated means.
+  if (!truncated && entries.length >= SEED_LIMITS.urlsPerDocument && matches.length > SEED_LIMITS.urlsPerDocument) truncated = true;
+  return { kind, entries, truncated, skipped };
+}
+
+/** The URL-only view of a sitemap, for callers that do not diff by lastmod. */
+export function parseSitemap(input, options = {}) {
+  const { kind, entries, truncated, skipped } = parseSitemapEntries(input, options);
+  return { kind, urls: entries.map(entry => entry.loc), truncated, skipped };
 }
 
 /** A sitemap entry must stand on its own as a public absolute URL. */
@@ -125,37 +229,42 @@ function absoluteEntry(value) {
   return validatePublicUrl(value);
 }
 
-/** The protocol's plain-text format: one absolute URL per line, no markup. */
+/** The protocol's plain-text format: one absolute URL per line, no markup. No lastmod. */
 function parseTextSitemap(text, { baseUrl, enforceScope, scope = 'path' }) {
-  const urls = [], skipped = {};
+  const entries = [], seen = new Set(), skipped = {};
   let truncated = false;
   for (const line of text.split(/\r?\n/u)) {
     const value = line.trim();
     if (!value || value.startsWith('#')) continue;
     // Anything with markup is not a text sitemap; do not half-read an unknown document.
     if (value.startsWith('<')) throw new SeedError('not_a_sitemap');
-    if (urls.length >= SEED_LIMITS.urlsPerDocument) { truncated = true; break; }
+    if (entries.length >= SEED_LIMITS.urlsPerDocument) { truncated = true; break; }
     const checked = absoluteEntry(value);
     if (!checked.valid) { skipped[checked.reason] = (skipped[checked.reason] ?? 0) + 1; continue; }
     if (enforceScope && baseUrl && !withinSitemapScope(checked.url, baseUrl, scope)) {
       skipped.out_of_sitemap_scope = (skipped.out_of_sitemap_scope ?? 0) + 1;
       continue;
     }
-    if (!urls.includes(checked.url)) urls.push(checked.url);
+    if (seen.has(checked.url)) continue;
+    seen.add(checked.url);
+    entries.push({ loc: checked.url, lastmod: null });
   }
-  if (!urls.length && !Object.keys(skipped).length) throw new SeedError('not_a_sitemap');
-  return { kind: 'urlset', urls, truncated, skipped };
+  if (!entries.length && !Object.keys(skipped).length) throw new SeedError('not_a_sitemap');
+  return { kind: 'urlset', entries, truncated, skipped };
 }
 
 /**
- * Walks a sitemap index to its leaf URLs, using a caller-supplied fetch so this module still
- * performs no I/O. `fetchDocument(url)` returns the body text or null when it could not be read.
+ * Walks a sitemap index to its leaf entries, using a caller-supplied fetch so this module
+ * still performs no I/O. `fetchDocument(url)` returns the body text or null when it could not
+ * be read. Leaf entries carry their declared lastmod; an index child carries the index's own
+ * lastmod, which describes the FILE rather than the URLs inside it, and so is not attached to
+ * anything downstream.
  *
  * A document we could not read is counted, never treated as a sitemap with no URLs — the same
  * rule extraction follows, for the same reason.
  */
 export async function expandSitemaps(roots, fetchDocument, { maxDepth = SEED_LIMITS.indexDepth, maxDocuments = SEED_LIMITS.sitemapDirectives, scope = 'path' } = {}) {
-  const urls = [], visited = new Set(), failures = {};
+  const entries = [], seenLocs = new Set(), visited = new Set(), failures = {};
   let documents = 0, truncated = false;
   const queue = roots.map(url => ({ url, depth: 0 }));
   while (queue.length) {
@@ -167,16 +276,20 @@ export async function expandSitemaps(roots, fetchDocument, { maxDepth = SEED_LIM
     const body = await fetchDocument(url);
     if (body === null || body === undefined) { failures.unreadable = (failures.unreadable ?? 0) + 1; continue; }
     let parsed;
-    try { parsed = parseSitemap(body, { baseUrl: url, scope }); }
+    try { parsed = parseSitemapEntries(body, { baseUrl: url, scope }); }
     catch (error) { failures[error.reason] = (failures[error.reason] ?? 0) + 1; continue; }
     if (parsed.truncated) truncated = true;
     if (parsed.kind === 'index') {
       // An index deeper than the limit is a loop or a mistake; stop rather than follow it.
       if (depth >= maxDepth) { failures.index_too_deep = (failures.index_too_deep ?? 0) + 1; continue; }
-      for (const child of parsed.urls) queue.push({ url: child, depth: depth + 1 });
+      for (const child of parsed.entries) queue.push({ url: child.loc, depth: depth + 1 });
       continue;
     }
-    for (const found of parsed.urls) if (!urls.includes(found)) urls.push(found);
+    for (const found of parsed.entries) {
+      if (seenLocs.has(found.loc)) continue;
+      seenLocs.add(found.loc);
+      entries.push(found);
+    }
   }
-  return { urls, documents, truncated, failures };
+  return { urls: entries.map(entry => entry.loc), entries, documents, truncated, failures };
 }

package/src/evidence-reference.js

@@ -0,0 +1,112 @@
+// The evidence reference (DP-0004-T21): one stable, linkable name for one observation.
+//
+// A report row says a link was present on a date. A reference is the part that lets a reader
+// point at the observation BEHIND that sentence — after the document was forwarded, after the
+// session that produced it is gone, and, for the repository mirror, after the hosted account
+// is cancelled. The buyer-lane demand this answers: an agency has to show a client WHY it
+// believes a placement state, not just that it does.
+//
+// Two storages, because that is where observations actually live:
+//
+//   hosted — the workspace's immutable observation row. Resolves through `locate_link`,
+//            `get_evidence` or `GET /v1/observations/{id}/evidence` while the workspace
+//            exists; the raw snapshot answers 410 EVIDENCE_EXPIRED once its lifecycle
+//            removes the bytes, and the reference still names the observation.
+//   mirror — the customer's repository observations file, keyed the way `observationKey`
+//            keys a row: which entry, when it was checked, and who ran the check. Resolves
+//            offline with `agentlinkops locate --ref …` from the repository alone.
+//
+// A reference is DERIVED, never minted. Every component comes from immutable observation
+// data, so the same observation produces the same reference on any machine on any day, and
+// nothing in a reference is live session state — which is what makes an export that carries
+// them survivable past cancelation. Anchoring (DP-0042-T07) will later make the content hash
+// independently verifiable; the reference is the handle that hash attaches to.
+export const EVIDENCE_REFERENCE_VERSION = 1;
+const PREFIX = `agentlinkops:evidence/${EVIDENCE_REFERENCE_VERSION}`;
+const ID = /^[\w-]{1,128}$/u;
+const ORIGINS = Object.freeze(['local', 'cloud']);
+
+const validTimestamp = value => typeof value === 'string' && value.length <= 64 && Number.isFinite(Date.parse(value));
+
+/** A reference to a hosted observation row. Null rather than a reference that cannot resolve. */
+export function hostedEvidenceReference(observationId) {
+  if (typeof observationId !== 'string' || !ID.test(observationId)) return null;
+  return `${PREFIX}/link/hosted/${observationId}`;
+}
+
+/**
+ * A reference to an observation in the repository mirror.
+ *
+ * `checkedAt` is the check's own timestamp (the cloud's, for cloud rows) rather than a local
+ * clock, so a replayed event produces the same reference. `origin` says who ran the check,
+ * because a local check and a cloud check of the same entry at the same instant are two real
+ * observations, not a duplicate.
+ */
+export function mirrorEvidenceReference(input = {}) {
+  const { entryId, checkedAt, origin } = input ?? {};
+  if (typeof entryId !== 'string' || !ID.test(entryId)) return null;
+  if (!ORIGINS.includes(origin)) return null;
+  if (!validTimestamp(checkedAt)) return null;
+  return `${PREFIX}/link/mirror/${entryId}/${origin}/${encodeURIComponent(checkedAt)}`;
+}
+
+/**
+ * Parses a reference back into what it names. Strict on purpose: a malformed reference is a
+ * reader's typo or a forged document, and resolving it by similarity would point at the wrong
+ * observation while appearing to verify the right one.
+ *
+ * Target-kind references are not part of this version; a future version adds them rather than
+ * this parser accepting a shape nothing resolves.
+ */
+export function parseEvidenceReference(reference) {
+  if (typeof reference !== 'string' || reference.length > 512) return null;
+  const parts = reference.split('/');
+  if (parts[0] !== 'agentlinkops:evidence' || parts[1] !== String(EVIDENCE_REFERENCE_VERSION)) return null;
+  if (parts[2] === 'index') {
+    if(parts[3]==='mirror' && parts.length===5 && /^[a-f0-9]{64}$/.test(parts[4]))return {v:1,subject:'index',storage:'mirror',receiptId:parts[4]};
+    if(parts[3]==='hosted' && parts.length===7 && ID.test(parts[4]) && ID.test(parts[5]) && /^[a-f0-9]{64}$/.test(parts[6]))return {v:1,subject:'index',storage:'hosted',projectId:parts[4],watchId:parts[5],receiptId:parts[6]};
+    return null;
+  }
+  if (parts[2] !== 'link') return null;
+  const [storage, ...rest] = parts.slice(3);
+  if (storage === 'hosted') {
+    const [observationId] = rest;
+    if (rest.length !== 1 || !ID.test(observationId ?? '')) return null;
+    return { v: EVIDENCE_REFERENCE_VERSION, subject: 'link', storage: 'hosted', observationId };
+  }
+  if (storage === 'mirror') {
+    const [entryId, origin, encodedAt] = rest;
+    if (rest.length !== 3 || !ID.test(entryId ?? '') || !ORIGINS.includes(origin ?? '')) return null;
+    let checkedAt;
+    try { checkedAt = decodeURIComponent(encodedAt ?? ''); } catch { return null; }
+    if (!validTimestamp(checkedAt)) return null;
+    return { v: EVIDENCE_REFERENCE_VERSION, subject: 'link', storage: 'mirror', entryId, checkedAt, origin };
+  }
+  return null;
+}
+
+/** How a holder resolves a reference, stated beside the reference rather than guessed at. */
+export function describeEvidenceReference(reference) {
+  const parsed = parseEvidenceReference(reference);
+  if (!parsed) return null;
+  if(parsed.subject==='index')return {reference,...parsed,resolves:parsed.storage==='mirror'?['agentlinkops locate --ref <reference> (offline exported index receipt)']:[`agentlinkops call get_index_observation --set projectId=${parsed.projectId} --set watchId=${parsed.watchId} --set receiptId=${parsed.receiptId}`],survives_cancelation:parsed.storage==='mirror'};
+  return {
+    reference,
+    ...parsed,
+    resolves: parsed.storage === 'hosted'
+      ? [
+        'agentlinkops call locate_link --set observationId=<id>  (positions and live-page handoff)',
+        'agentlinkops call get_evidence --set subject=link --set observationId=<id>  (retained snapshot; 410 once expired)',
+        'GET /v1/observations/<id>/evidence  (HTTP, workspace credentials)',
+      ]
+      : ['agentlinkops locate --ref <reference>  (offline, from the repository ledger and observations)'],
+    survives_cancelation: parsed.storage === 'mirror',
+  };
+}
+
+export function indexEvidenceReference({receiptId,projectId,watchId,storage='mirror'}={}){
+ if(!/^[a-f0-9]{64}$/.test(receiptId??''))return null;
+ if(storage==='mirror')return `${PREFIX}/index/mirror/${receiptId}`;
+ if(storage==='hosted'&&ID.test(projectId??'')&&ID.test(watchId??''))return `${PREFIX}/index/hosted/${projectId}/${watchId}/${receiptId}`;
+ return null;
+}

package/src/index-observation.js

@@ -0,0 +1,161 @@
+// Pure index-observation semantics. No fetch, OAuth, scraping, quota mutation,
+// persistence or hosted feature activation occurs in this module.
+export const INDEX_OBSERVATION_VERSION = 1;
+export const INDEX_CONFIRMATION = Object.freeze({ probes: 2, minProbeIntervalMs: 30 * 60_000, graceMs: 24 * 60 * 60_000 });
+export const URL_INSPECTION_LIMITS = Object.freeze({ propertyPerDay: 2000, propertyPerMinute: 600, projectPerDay: 10_000_000, projectPerMinute: 15_000 });
+const positiveCoverage = new Set(['Submitted and indexed', 'Indexed, not submitted in sitemap', 'Indexed']);
+const fail = code => { throw Object.assign(new Error(code), { code }); };
+const text = value => typeof value === 'string' && value.length ? value : null;
+function timestamp(value) {
+  if (typeof value !== 'string' || !/^\d{4}-\d\d-\d\dT.*(?:Z|[+-]\d\d:\d\d)$/u.test(value) || !Number.isFinite(Date.parse(value))) fail('INVALID_INDEX_TIMESTAMP');
+  return new Date(value).toISOString();
+}
+function url(value) {
+  let parsed; try { parsed = new URL(value); } catch { fail('INVALID_INDEX_URL'); }
+  if (!['https:', 'http:'].includes(parsed.protocol) || parsed.username || parsed.password || parsed.hash) fail('INVALID_INDEX_URL');
+  return parsed.href;
+}
+export function validateInspectionProperty(inspectionUrl, siteUrl, grantedProperties) {
+  const inspected = new URL(url(inspectionUrl));
+  if (!Array.isArray(grantedProperties) || !grantedProperties.includes(siteUrl)) fail('INDEX_PROPERTY_NOT_GRANTED');
+  if (typeof siteUrl !== 'string') fail('INVALID_INDEX_PROPERTY');
+  if (siteUrl.startsWith('sc-domain:')) {
+    const domain = siteUrl.slice(10);
+    if (!/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/u.test(domain) || domain.includes('..')) fail('INVALID_INDEX_PROPERTY');
+    if (inspected.hostname !== domain && !inspected.hostname.endsWith(`.${domain}`)) fail('INDEX_URL_OUTSIDE_PROPERTY');
+  } else {
+    const property = new URL(url(siteUrl));
+    if (!siteUrl.endsWith('/') || property.search) fail('INVALID_INDEX_PROPERTY');
+    if (inspected.origin !== property.origin || !inspected.pathname.startsWith(property.pathname)) fail('INDEX_URL_OUTSIDE_PROPERTY');
+  }
+  // Keep the exact granted property string; do not silently fix a missing slash.
+  return siteUrl;
+}
+function envelope({ observationId, inspectionUrl, checkedAt, payload, lane, backend, sourceKey }) {
+  if (!text(observationId) || observationId.length > 128) fail('INVALID_INDEX_OBSERVATION_ID');
+  let raw; try { raw = payload === undefined ? null : JSON.parse(JSON.stringify(payload)); } catch { fail('INVALID_INDEX_RESPONSE'); }
+  if (new TextEncoder().encode(JSON.stringify(raw)).byteLength > 262144) fail('INDEX_RESPONSE_EXCEEDS_BOUND');
+  return { version: INDEX_OBSERVATION_VERSION, observation_id: observationId, url: url(inspectionUrl), checked_at: timestamp(checkedAt),
+    lane, backend, source_key: sourceKey, tier: 'unknown', confidence: 'unknown', reason: 'invalid_response',
+    raw_response: raw, last_crawl_at: null, live_page_test: false, submission_requested: false };
+}
+export function gscIndexObservation(input) {
+  const siteUrl = validateInspectionProperty(input.inspectionUrl, input.siteUrl, input.grantedProperties);
+  const observation = envelope({ ...input, lane: 'gsc', backend: 'google_url_inspection', sourceKey: `gsc:${siteUrl}` });
+  observation.property = siteUrl;
+  const response = observation.raw_response;
+  const result = response?.inspectionResult?.indexStatusResult;
+  if (response?.error) return { ...observation, reason: 'inspection_request_failed' };
+  if (!result || typeof result !== 'object' || Array.isArray(result)) return observation;
+  const tuple = Object.fromEntries(['verdict', 'coverageState', 'indexingState', 'robotsTxtState', 'pageFetchState', 'googleCanonical', 'userCanonical', 'lastCrawlTime', 'crawledAs'].map(key => [key, text(result[key])]));
+  observation.inspection = tuple;
+  observation.reason = tuple.coverageState ?? 'coverage_reason_missing';
+  observation.last_crawl_at = tuple.lastCrawlTime; // Provider time is not our retrieval time.
+  observation.inspection_result_link = text(response.inspectionResult.inspectionResultLink);
+  if (!tuple.coverageState) return observation;
+  if (tuple.verdict === 'PASS') {
+    // coverageState is human-readable and can be localized. v1 admits only these
+    // reviewed en-US positive labels; unknown/new labels fail closed.
+    let canonicalMatches = false;
+    try { canonicalMatches = url(tuple.googleCanonical) === observation.url; } catch { /* missing or invalid is unknown */ }
+    if (positiveCoverage.has(tuple.coverageState) && tuple.indexingState === 'INDEXING_ALLOWED' && tuple.pageFetchState === 'SUCCESSFUL' && canonicalMatches) {
+      return { ...observation, tier: 'indexed', confidence: 'google_index_snapshot' };
+    }
+    return { ...observation, reason: 'inconsistent_or_unsupported_index_tuple' };
+  }
+  if (['NEUTRAL', 'FAIL'].includes(tuple.verdict) && !positiveCoverage.has(tuple.coverageState)) {
+    return { ...observation, tier: 'not_indexed', confidence: 'google_index_snapshot' };
+  }
+  return { ...observation, reason: 'unspecified_or_inconsistent_index_verdict' };
+}
+export function licensedSiteIndexObservation(input) {
+  if (!text(input.provider) || input.provider.length > 128) fail('INVALID_INDEX_PROVIDER');
+  const inspected = url(input.inspectionUrl);
+  if (input.query !== `site:${inspected}`) fail('INDEX_QUERY_SCOPE_MISMATCH');
+  if (!text(input.country) || !text(input.language)) fail('INDEX_LOCALE_REQUIRED');
+  const sourceKey = JSON.stringify(['licensed_site', input.provider, input.query, input.country, input.language]);
+  const observation = envelope({ ...input, lane: 'licensed_site', backend: input.provider, sourceKey });
+  observation.query = input.query; observation.country = input.country; observation.language = input.language;
+  const result = observation.raw_response;
+  if (result?.success !== true || result.complete !== true || !Array.isArray(result.results)) return observation;
+  const urls = [];
+  for (const row of result.results) {
+    try { urls.push(url(row?.url)); } catch { return { ...observation, reason: 'invalid_result_url' }; }
+  }
+  return { ...observation, tier: urls.includes(inspected) ? 'likely_indexed' : 'not_found_in_site_query',
+    confidence: 'search_result_sample', reason: urls.includes(inspected) ? 'exact_url_returned' : 'exact_url_not_returned' };
+}
+
+export function transitionIndexObservation(previous, observation, policy = INDEX_CONFIRMATION) {
+  if (observation?.version !== INDEX_OBSERVATION_VERSION || !['gsc', 'licensed_site'].includes(observation.lane)
+    || !['indexed', 'not_indexed', 'likely_indexed', 'not_found_in_site_query', 'unknown'].includes(observation.tier)
+    || !text(observation.observation_id) || !text(observation.source_key)) fail('INVALID_INDEX_OBSERVATION');
+  if (observation.lane !== 'gsc' && ['indexed', 'not_indexed'].includes(observation.tier)) fail('INVALID_INDEX_CONFIDENCE');
+  const at = timestamp(observation.checked_at);
+  if (!Number.isInteger(policy.probes) || policy.probes < 2 || policy.probes > 10 || !Number.isFinite(policy.minProbeIntervalMs)
+    || policy.minProbeIntervalMs < 1 || !Number.isFinite(policy.graceMs) || policy.graceMs < policy.minProbeIntervalMs) fail('INVALID_INDEX_CONFIRMATION_POLICY');
+  if (previous && (previous.url !== observation.url || previous.source_key !== observation.source_key)) fail('INDEX_STATE_SCOPE_MISMATCH');
+  const state = previous ? structuredClone(previous) : { version: 1, url: observation.url, source_key: observation.source_key,
+    confirmed: null, pending_negative: null, latest: null };
+  if (state.latest && (state.latest.observation_id === observation.observation_id || state.confirmed?.observation_id === observation.observation_id
+    || state.pending_negative?.observation_ids.includes(observation.observation_id) || Date.parse(at) <= Date.parse(state.latest.checked_at))) {
+    return { state, event: null, ignored: 'duplicate_or_out_of_order' };
+  }
+  state.latest = structuredClone(observation);
+  if (observation.tier === 'unknown' || observation.lane !== 'gsc') return { state, event: null };
+  if (observation.tier === 'indexed') {
+    const before = state.confirmed;
+    state.confirmed = { tier: 'indexed', observed_at: at, observation_id: observation.observation_id, reason: observation.reason };
+    state.pending_negative = null;
+    return { state, event: before?.tier === 'not_indexed' ? { type: 'index.reindexed',
+      id: `index.reindexed:${before.observation_id}:${observation.observation_id}`, url: observation.url,
+      source_key: observation.source_key, before: before.tier, after: 'indexed', observation_ids: [before.observation_id, observation.observation_id] } : null };
+  }
+  if (observation.tier !== 'not_indexed' || state.confirmed?.tier === 'not_indexed') return { state, event: null };
+  const pending = state.pending_negative ?? { first_at: at, last_counted_at: at, observation_ids: [observation.observation_id] };
+  if (state.pending_negative && Date.parse(at) - Date.parse(pending.last_counted_at) >= policy.minProbeIntervalMs) {
+    if (pending.observation_ids.length < policy.probes) pending.observation_ids.push(observation.observation_id);
+    pending.last_counted_at = at;
+  }
+  state.pending_negative = pending;
+  if (pending.observation_ids.length < policy.probes || Date.parse(at) - Date.parse(pending.first_at) < policy.graceMs) return { state, event: null };
+  const before = state.confirmed;
+  state.confirmed = { tier: 'not_indexed', observed_at: at, observation_id: observation.observation_id, reason: observation.reason };
+  state.pending_negative = null;
+  return { state, event: before?.tier === 'indexed' ? { type: 'index.deindexed',
+    id: `index.deindexed:${pending.observation_ids[0]}:${observation.observation_id}`, url: observation.url,
+    source_key: observation.source_key, before: before.tier, after: 'not_indexed',
+    observation_ids: [...new Set([...pending.observation_ids, observation.observation_id])], first_negative_at: pending.first_at } : null };
+}
+
+// Advisory arithmetic only: a future transactional scheduler must reserve all
+// four budgets atomically. This function cannot admit a network call.
+export function inspectionQuotaHeadroom(usage, limits = URL_INSPECTION_LIMITS) {
+  return Math.min(...Object.keys(URL_INSPECTION_LIMITS).map(key => {
+    const row = usage?.[key];
+    if (!row || !Number.isSafeInteger(row.attempted) || row.attempted < 0 || !Number.isSafeInteger(row.reserved) || row.reserved < 0
+      || !Number.isSafeInteger(limits[key]) || limits[key] < 0) fail('INVALID_INDEX_QUOTA_USAGE');
+    return Math.max(0, limits[key] - row.attempted - row.reserved);
+  }));
+}
+
+// Compare dated evidence; never overwrite either source or select a winner.
+export function reconcileIndexObservations(observations,{maxSkewMs=86400000}={}){
+ if(!Array.isArray(observations)||observations.length>100||!Number.isFinite(maxSkewMs)||maxSkewMs<0)fail('INVALID_INDEX_RECONCILIATION');
+ const bySource=new Map();
+ for(const observation of observations){
+  if(!observation?.url||!observation.source_key)fail('INVALID_INDEX_OBSERVATION');timestamp(observation.checked_at);
+  const before=bySource.get(observation.source_key);if(!before||observation.checked_at>before.checked_at||(observation.checked_at===before.checked_at&&observation.observation_id>before.observation_id))bySource.set(observation.source_key,observation);
+ }
+ const rows=[...bySource.values()].sort((a,b)=>a.source_key.localeCompare(b.source_key));
+ if(new Set(rows.map(r=>r.url)).size>1)fail('INDEX_STATE_SCOPE_MISMATCH');
+ const comparisons=[];
+ for(let i=0;i<rows.length;i++)for(let j=i+1;j<rows.length;j++){
+  const a=rows[i],b=rows[j];if(a.lane===b.lane)continue;
+  const signs={indexed:1,likely_indexed:1,not_indexed:-1,not_found_in_site_query:-1};
+  const skew=Math.abs(Date.parse(a.checked_at)-Date.parse(b.checked_at));
+  const finding=skew>maxSkewMs?'not_comparable':!signs[a.tier]||!signs[b.tier]?'inconclusive':signs[a.tier]===signs[b.tier]?'agreement':'disagreement';
+  comparisons.push({finding,resolution:'unresolved',observation_ids:[a.observation_id,b.observation_id],source_keys:[a.source_key,b.source_key],checked_at:[a.checked_at,b.checked_at],tiers:[a.tier,b.tier],time_skew_ms:skew,negative_search_result_is_not_proof:true});
+ }
+ return {version:1,url:rows[0]?.url??null,comparisons,combined_verdict:null};
+}