agentlinkops 0.6.1 → 0.6.3

package/src/locator.js

@@ -0,0 +1,112 @@
+// The evidence locator (DP-0004-T05): where a recorded link is, without rendering anything.
+//
+// An agent that has been told a link exists wants two things: to see the occurrence we recorded,
+// and to go look at the page. Neither needs the captured HTML, and **the captured HTML must
+// never be the thing we render.** The evidence route already refuses to: it serves the snapshot
+// as `application/json`, as an attachment, `no-store`, under `default-src 'none'; sandbox`. This
+// module gives the other half — a position and a way to open the real page — from metadata
+// alone.
+//
+// Two distinctions run through everything here, and both are about not lying by omission.
+//
+//   **The locator points into the DOCUMENT WE FETCHED, not into the live page.** Line 1,069 of
+//   what we captured on the 11th may be line 900 today, or gone. It is a position in evidence,
+//   which is exactly what makes it useful for reading the evidence and useless for anything else.
+//
+//   **Static and rendered evidence stay apart.** A static observation proves the link was in the
+//   served HTML. It says nothing about whether a reader sees it, and a locator that blurred the
+//   two would be the most convincing place to do it.
+export const LOCATOR_LIMITS = Object.freeze({ occurrences: 50, fragmentChars: 300, contextChars: 320 });
+
+/**
+ * A text fragment that makes a browser scroll to and highlight the anchor on the LIVE page.
+ *
+ * `-`, `,` and `&` are delimiters inside the fragment syntax, so they are percent-encoded beyond
+ * what `encodeURIComponent` does. Text spanning a line break cannot be matched as one run, so
+ * anything with a newline is refused rather than silently truncated into a fragment that lands
+ * somewhere else on the page.
+ */
+export function textFragment(sourceUrl, text) {
+  const phrase = String(text ?? '').trim();
+  if (!phrase || /[\r\n]/u.test(phrase)) return null;
+  // Too short to be distinctive, and a one-character fragment highlights the wrong thing.
+  if (phrase.length < 3) return null;
+  const trimmed = phrase.slice(0, LOCATOR_LIMITS.fragmentChars);
+  const encoded = encodeURIComponent(trimmed).replace(/-/gu, '%2D').replace(/,/gu, '%2C').replace(/&/gu, '%26');
+  let url;
+  try { url = new URL(sourceUrl); } catch { return null; }
+  if (!/^https?:$/u.test(url.protocol)) return null;
+  // A text fragment lives after any existing fragment, not instead of it.
+  const existing = url.hash.startsWith('#') ? url.hash.slice(1) : '';
+  url.hash = `${existing}:~:text=${encoded}`;
+  return url.href;
+}
+
+const followedOf = (rel, directives) => {
+  const tokens = (rel ?? []).map(value => String(value).toLowerCase());
+  if (tokens.includes('nofollow') || tokens.includes('sponsored') || tokens.includes('ugc')) return false;
+  if (directives?.nofollow) return false;
+  return true;
+};
+
+/**
+ * Turns a stored observation into somewhere to look.
+ *
+ * Takes the observation's own `result` — the object the verifier produced, which both the cloud
+ * and `agentlinkops check` store — so this works identically on a row from either.
+ */
+export function locateOccurrences(observation, { limit = LOCATOR_LIMITS.occurrences } = {}) {
+  const result = observation?.result ?? observation ?? {};
+  const occurrences = Array.isArray(result.occurrences) ? result.occurrences : [];
+  const sourceUrl = result.finalUrl ?? result.sourceUrl ?? null;
+  const rendered = result.evidence?.rendered === true;
+  return {
+    v: 1,
+    observation_id: observation?.id ?? null,
+    state: result.state ?? null,
+    reason: result.reason ?? null,
+    source_url: result.sourceUrl ?? null,
+    // Where we actually read, which is not always where we were sent.
+    final_url: sourceUrl,
+    target_url: result.targetUrl ?? null,
+    observed_at: result.checkedAt ?? observation?.checked_at ?? null,
+    checker_version: result.evidence?.checkerVersion ?? observation?.checker_version ?? null,
+    // Named rather than implied. A static observation is not a claim about what a reader sees.
+    evidence_method: rendered ? 'rendered' : 'static_html',
+    rendered,
+    evidence: {
+      key: observation?.evidence_key ?? null,
+      sha256: result.evidence?.sha256 ?? null,
+      bytes: result.evidence?.bytes ?? null,
+      // A reader has to know the snapshot may be gone before they plan around fetching it.
+      retrieval: observation?.evidence_key ? 'GET /v1/observations/{id}/evidence, 410 once expired' : null,
+    },
+    occurrence_count: result.occurrenceCount ?? occurrences.length,
+    truncated: occurrences.length > limit || result.occurrencesTruncated === true,
+    occurrences: occurrences.slice(0, limit).map((item, index) => ({
+      index,
+      href: item.href ?? null,
+      resolved_target: item.targetUrl ?? null,
+      anchor: item.anchor ?? null,
+      rel: item.rel ?? [],
+      followed: followedOf(item.rel, result.directives),
+      context: typeof item.context === 'string' ? item.context.slice(0, LOCATOR_LIMITS.contextChars) : null,
+      // A position in the EVIDENCE, labelled as one.
+      in_captured_document: item.locator ?? null,
+      // A position in the LIVE page, which is where a human should actually look.
+      open_at: textFragment(sourceUrl, item.anchor) ?? textFragment(sourceUrl, item.context) ?? sourceUrl,
+    })),
+    page_directives: result.directives
+      ? { noindex: result.directives.noindex ?? null, nofollow: result.directives.nofollow ?? null, indexing_status: result.directives.indexingStatus ?? null }
+      : null,
+    // Said once, plainly, on every locate. The alternative is a reader who opens the live page,
+    // sees something different, and concludes the tool is wrong.
+    notes: [
+      'Positions are offsets into the captured document, not into the page as it is now.',
+      rendered
+        ? 'This observation was rendered; occurrences may include links written by the page\'s own scripts.'
+        : 'Static HTML observation. JavaScript execution and visual visibility were not checked.',
+      'open_at points at the live page with a text fragment. The page may have changed since it was observed.',
+    ],
+  };
+}

package/src/tool-views.js

@@ -3,12 +3,16 @@
 // script, the MCP router, the CLI index and the generators all read the same definitions.
 // Rules: docs/technical/AGENT-SURFACE-CONTRACT.md (DP-0036).
 
-export const TOOLSETS = ['monitoring','evidence','discovery','library','competitors','reports','workspace','notifications','webhooks'];
+export const TOOLSETS = ['monitoring','evidence','discovery','library','competitors','reports','workspace','notifications','webhooks','admission','disavow','lifecycle','digests'];
 
 // One line per toolset: what the group is for. Printed by the site brief, the REST index and the
 // catalog snapshot so the same wording reaches every client; a toolset without a summary is a
 // gate failure in the measurement script.
 export const TOOLSET_SUMMARIES = {
+  admission: 'Project rules, URL previews and retained admission decisions.',
+  disavow: 'Propose, review and export website disavow rules.',
+  lifecycle: 'Placement costs, expiry, renewal events and currency reports.',
+  digests: 'Own-address digest preferences, delivery history and exact events.',
   monitoring: 'Watch earned links and destination URLs: create, list, update, import, export, recheck.',
   evidence: 'What a check observed: histories, snapshots, change feeds, check jobs, published contacts.',
   discovery: 'Import candidate rows, read stored runs, verify selected candidates, enroll them.',
@@ -51,7 +55,7 @@ export function oneLiner(description, max = 60) {
 }
 
 // entries: [{ name, description, toolset, tier, readOnly }]. Grouped, one line per command.
-export function manifestText(entries, { max = 60 } = {}) {
+export function manifestText(entries, { max = 36 } = {}) {
   const groups = new Map(TOOLSETS.map(t => [t, []]));
   for (const entry of entries) (groups.get(entry.toolset) ?? groups.set(entry.toolset, []).get(entry.toolset)).push(entry);
   const lines = ['Commands by toolset (name: what it does). Use search_tools or describe_tools for the full schema before calling.'];

package/src/verifier/fetch.js

@@ -115,6 +115,13 @@ export function publicFetcher(options, signal) {
     try {
       response = await withAbort(Promise.resolve().then(() => {
         if (signal.aborted) throw new VerificationError('timeout');
+        // Conditional headers ride only on the request that earned them. A validator belongs
+        // to one resource; across a redirect hop it would be re-sent to a different one, so
+        // the caller drops validators after the first hop and the redirect loop revalidates
+        // the destination from scratch rather than trusting a moved resource's old validators.
+        const conditional = {};
+        if (context.kind === 'source' && context.validators?.etag) conditional['If-None-Match'] = context.validators.etag;
+        if (context.kind === 'source' && context.validators?.lastModified) conditional['If-Modified-Since'] = context.validators.lastModified;
         return fetchImpl(checked.url, {
           method: 'GET',
           redirect: 'manual',
@@ -122,7 +129,8 @@ export function publicFetcher(options, signal) {
           signal,
           headers: {
             'User-Agent': userAgent,
-            Accept: context.kind === 'robots' ? 'text/plain' : 'text/html',
+            Accept: context.kind === 'robots' ? 'text/plain' : context.accept ?? 'text/html',
+            ...conditional,
           },
         });
       }), signal);
@@ -239,8 +247,23 @@ export function publicFetcher(options, signal) {
       if (signal.aborted) throw new VerificationError('timeout');
       return robots;
     },
-    async source(initialUrl, trace) {
+    /**
+     * Fetches one source document. `{ validators }` turns this into a conditional GET: the
+     * caller passes the ETag/Last-Modified a previous fetch earned, and a 304 comes back as
+     * `{ notModified: true }` rather than an error. A 304 we did not ask for stays an error —
+     * the server revalidating nobody's cache is a defect, not evidence of anything.
+     *
+     * `{ document: true }` fetches a feed or sitemap instead of a page: the Accept header and
+     * the content-type gate admit the XML family rather than HTML. Everything else — robots per
+     * hop, public-destination screening, the redirect budget — is identical, because a feed is
+     * fetched from the same publishers under the same politeness rules as a page.
+     */
+    async source(initialUrl, trace, { validators = null, document = false } = {}) {
+      const accept = document
+        ? 'application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.9, text/plain;q=0.8'
+        : 'text/html';
       let url = initialUrl;
+      let revalidators = validators;
       const seen = new Set();
       for (let redirects = 0; redirects <= maxRedirects; redirects++) {
         trace.finalUrl = url;
@@ -248,7 +271,7 @@ export function publicFetcher(options, signal) {
         seen.add(url);
         const robots = await checkRobots(url, trace);
         trace.robots = robots;
-        const response = await request(url, { kind: 'source', crawlDelaySeconds: robots.crawlDelaySeconds });
+        const response = await request(url, { kind: 'source', crawlDelaySeconds: robots.crawlDelaySeconds, validators: revalidators, accept });
         trace.httpStatus = response.status;
         trace.sourceResponse = { url, httpStatus: response.status,
           contentType: /^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+/iu.exec(response.headers.get('content-type') ?? '')?.[0].slice(0, 128) ?? null,
@@ -269,6 +292,9 @@ export function publicFetcher(options, signal) {
           if (!next.valid) throw new VerificationError(`unsafe_redirect:${next.reason}`);
           trace.redirects.push({ from: url, to: next.url, status: response.status });
           url = next.url;
+          // Validators were earned for the pre-redirect resource; the destination revalidates
+          // from scratch (see the conditional-header note in `request`).
+          revalidators = null;
           continue;
         }
         if ([404, 410].includes(response.status)) {
@@ -277,7 +303,10 @@ export function publicFetcher(options, signal) {
         }
         if (response.status === 304) {
           cancelBody(response);
-          throw new VerificationError('not_modified_without_baseline');
+          if (!revalidators) throw new VerificationError('not_modified_without_baseline');
+          return { response, body: null, notModified: true,
+            validators: { etag: response.headers.get('etag') ?? revalidators.etag ?? null,
+              lastModified: response.headers.get('last-modified') ?? revalidators.lastModified ?? null } };
         }
         if (response.status !== 200) {
           cancelBody(response);
@@ -290,12 +319,19 @@ export function publicFetcher(options, signal) {
             retryAfterSeconds(response.headers.get('retry-after')) === null ? {}
               : { retryAfterSeconds: retryAfterSeconds(response.headers.get('retry-after')) });
         }
-        if (!/^text\/html(?:;|$)/iu.test(response.headers.get('content-type') ?? '')) {
+        const contentType = response.headers.get('content-type') ?? '';
+        // The document gate admits the XML family (`application/xml`, `text/xml`,
+        // `application/rss+xml`, `application/atom+xml`) and plain text (the sitemap protocol's
+        // text format). It is still a gate: a feed URL serving HTML is a page wearing a feed's
+        // name, and parsing it as a feed would read markup as entries.
+        const expected = document ? /(?:[+/]xml|text\/plain)(?:\s*;|$)/iu : /^text\/html(?:;|$)/iu;
+        if (!expected.test(contentType)) {
           cancelBody(response);
           throw new VerificationError('unsupported_content_type');
         }
         const body = await readBounded(response, maxBytes, signal);
-        return { response, body };
+        return { response, body, notModified: false,
+          validators: { etag: response.headers.get('etag'), lastModified: response.headers.get('last-modified') } };
       }
       throw new VerificationError('redirect_limit');
     },

package/src/citations/browser/gen/core/providers/_shared/google.js

@@ -1,4 +0,0 @@
-const GOOGLE_CONSENT_SELECTOR = "button#L2AGLb, button#W0wltc, form[action*='consent.google.com'] button";
-export {
-  GOOGLE_CONSENT_SELECTOR
-};

package/src/citations/browser/gen/lib/browser/playwrightCompat.js

@@ -1,321 +0,0 @@
-import { runPageDomOp } from "./domOps.js";
-class PlaywrightWorkerCompat {
-  constructor(worker) {
-    this.worker = worker;
-  }
-  worker;
-  async evaluate(script) {
-    return await this.worker.evaluate(script);
-  }
-}
-class PlaywrightLocatorCompat {
-  constructor(locator) {
-    this.locator = locator;
-  }
-  locator;
-  wrap(next) {
-    return new PlaywrightLocatorCompat(next);
-  }
-  count() {
-    return this.locator.count();
-  }
-  nth(index) {
-    return this.wrap(this.locator.nth(index));
-  }
-  first() {
-    return this.wrap(this.locator.first());
-  }
-  last() {
-    return this.wrap(this.locator.last());
-  }
-  filter(options) {
-    return this.wrap(this.locator.filter({ hasText: options.hasText }));
-  }
-  getByText(text) {
-    return this.wrap(this.locator.getByText(text));
-  }
-  isVisible(options) {
-    return this.locator.isVisible(options);
-  }
-  isEnabled() {
-    return this.locator.isEnabled();
-  }
-  focus() {
-    return this.locator.focus();
-  }
-  boundingBox() {
-    return this.locator.boundingBox();
-  }
-  scrollIntoViewIfNeeded() {
-    return this.locator.scrollIntoViewIfNeeded();
-  }
-  click(options) {
-    return this.locator.click(options);
-  }
-  press(key, options) {
-    return this.locator.press(key, options);
-  }
-  waitFor(options) {
-    return this.locator.waitFor({
-      timeout: options?.timeout,
-      state: options?.state
-    });
-  }
-  async readInputValue() {
-    return await this.locator.evaluate((element) => {
-      if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
-        return element.value;
-      }
-      if (element instanceof HTMLElement) {
-        return element.innerText || element.textContent || "";
-      }
-      return "";
-    });
-  }
-  async setInputValue(value) {
-    await this.locator.evaluate((element, nextValue) => {
-      if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
-        const proto = Object.getPrototypeOf(element);
-        const descriptor = Object.getOwnPropertyDescriptor(proto, "value");
-        if (descriptor?.set) {
-          descriptor.set.call(element, nextValue);
-        } else {
-          element.value = nextValue;
-        }
-        element.dispatchEvent(new Event("input", { bubbles: true }));
-        element.dispatchEvent(new Event("change", { bubbles: true }));
-        return;
-      }
-      if (element instanceof HTMLElement) {
-        element.focus();
-        element.innerText = nextValue;
-        element.dispatchEvent(new Event("input", { bubbles: true }));
-        element.dispatchEvent(new Event("change", { bubbles: true }));
-      }
-    }, value);
-  }
-  async getEditableState() {
-    return await this.locator.evaluate((element) => {
-      function hasHiddenAncestor(target) {
-        let current = target;
-        while (current) {
-          if (current.hidden || current.getAttribute("aria-hidden") === "true" || current.hasAttribute("inert")) {
-            return true;
-          }
-          const style = window.getComputedStyle(current);
-          if (style.display === "none" || style.visibility === "hidden" || style.visibility === "collapse" || style.opacity === "0" || style.pointerEvents === "none") {
-            return true;
-          }
-          current = current.parentElement;
-        }
-        return false;
-      }
-      function acceptsTextInput(target) {
-        if (target instanceof HTMLTextAreaElement) {
-          return true;
-        }
-        if (target instanceof HTMLInputElement) {
-          const blockedTypes = /* @__PURE__ */ new Set([
-            "hidden",
-            "button",
-            "checkbox",
-            "color",
-            "file",
-            "image",
-            "radio",
-            "range",
-            "reset",
-            "submit"
-          ]);
-          return !blockedTypes.has(target.type);
-        }
-        return target.isContentEditable || target.getAttribute("contenteditable") === "true";
-      }
-      function isEnabled(target) {
-        if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) {
-          return !target.disabled && !target.readOnly;
-        }
-        return target.getAttribute("aria-disabled") !== "true" && !target.hasAttribute("disabled") && target.getAttribute("contenteditable") !== "false";
-      }
-      if (!(element instanceof HTMLElement)) {
-        return {
-          connected: false,
-          visible: false,
-          editable: false,
-          enabled: false,
-          acceptsTextInput: false
-        };
-      }
-      const rect = element.getBoundingClientRect();
-      const clientRects = element.getClientRects();
-      const visibleByBrowser = typeof element.checkVisibility === "function" ? element.checkVisibility({
-        checkOpacity: true,
-        checkVisibilityCSS: true
-      }) : true;
-      const visible = element.isConnected && visibleByBrowser && !hasHiddenAncestor(element) && clientRects.length > 0 && rect.width > 0 && rect.height > 0;
-      const enabled = isEnabled(element);
-      const textInput = acceptsTextInput(element);
-      return {
-        connected: element.isConnected,
-        visible,
-        editable: visible && enabled && textInput,
-        enabled,
-        acceptsTextInput: textInput
-      };
-    });
-  }
-  async dispatchClick() {
-    await this.locator.evaluate((element) => {
-      if (!(element instanceof HTMLElement)) return;
-      element.dispatchEvent(
-        new MouseEvent("click", {
-          bubbles: true,
-          cancelable: true,
-          composed: true,
-          view: window
-        })
-      );
-    });
-  }
-}
-class PlaywrightBrowserContextCompat {
-  constructor(context) {
-    this.context = context;
-    const existingPages = this.context.pages();
-    this.initialPage = existingPages.length === 1 && existingPages[0]?.url() === "about:blank" ? existingPages[0] : null;
-  }
-  context;
-  pageMap = /* @__PURE__ */ new WeakMap();
-  initialPage;
-  wrapPage(page) {
-    const existing = this.pageMap.get(page);
-    if (existing) return existing;
-    const wrapped = new PlaywrightPageCompat(page, this);
-    this.pageMap.set(page, wrapped);
-    return wrapped;
-  }
-  getRawContext() {
-    return this.context;
-  }
-  getBrowser() {
-    const browser = this.context.browser();
-    if (browser) return browser;
-    return {
-      version: () => "Camoufox",
-      close: () => this.context.close()
-    };
-  }
-  async newPage() {
-    if (this.initialPage) {
-      const page = this.initialPage;
-      this.initialPage = null;
-      return this.wrapPage(page);
-    }
-    return this.wrapPage(await this.context.newPage());
-  }
-  async close() {
-    await this.context.close();
-  }
-  async storageState(options) {
-    await this.context.storageState(options);
-  }
-  async addInitScript(script) {
-    await this.context.addInitScript(script);
-  }
-  on(event, listener) {
-    this.context.on(event, (page) => {
-      listener(this.wrapPage(page));
-    });
-  }
-}
-class PlaywrightPageCompat {
-  constructor(page, owner) {
-    this.page = page;
-    this.owner = owner;
-    this.mouse = this.page.mouse;
-    this.keyboard = this.page.keyboard;
-  }
-  page;
-  owner;
-  mouse;
-  keyboard;
-  getRawPage() {
-    return this.page;
-  }
-  async goto(url, options) {
-    await this.page.goto(url, options);
-  }
-  async evaluate(pageFunction, arg) {
-    return await this.page.evaluate(pageFunction, arg);
-  }
-  url() {
-    return this.page.url();
-  }
-  async getUrl() {
-    return this.page.url();
-  }
-  waitForTimeout(ms) {
-    return this.page.waitForTimeout(ms);
-  }
-  waitForLoadState(state, options) {
-    return this.page.waitForLoadState(state, options);
-  }
-  async waitForSelector(selector, options) {
-    await this.page.waitForSelector(selector, options ?? {});
-  }
-  locator(selector) {
-    return new PlaywrightLocatorCompat(this.page.locator(selector));
-  }
-  close() {
-    return this.page.close();
-  }
-  setDefaultTimeout(ms) {
-    this.page.setDefaultTimeout(ms);
-  }
-  setDefaultNavigationTimeout(ms) {
-    this.page.setDefaultNavigationTimeout(ms);
-  }
-  on(event, listener) {
-    if (event === "console") {
-      this.page.on(event, listener);
-      return;
-    }
-    this.page.on(event, (worker) => {
-      listener(
-        new PlaywrightWorkerCompat(worker)
-      );
-    });
-  }
-  context() {
-    return this.owner;
-  }
-  viewportSize() {
-    return this.page.viewportSize();
-  }
-  async runDomOp(operation, params) {
-    return await runPageDomOp(
-      this.page,
-      operation,
-      params ?? {}
-    );
-  }
-  async ping() {
-    try {
-      await Promise.race([
-        this.page.evaluate(() => true),
-        new Promise(
-          (_, reject) => setTimeout(() => reject(new Error("page ping timeout")), 5e3)
-        )
-      ]);
-      return true;
-    } catch {
-      return false;
-    }
-  }
-  async screenshot(options) {
-    return await this.page.screenshot(options ?? {});
-  }
-}
-export {
-  PlaywrightBrowserContextCompat
-};