agentlinkops 0.5.0

package/cli/gsc-browser.js

@@ -0,0 +1,239 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { createHash } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { parseArgs } from './args.js';
+import { parseCsv, CsvError } from './adapters/csv.js';
+
+export const GSC_BROWSER_HELP = `agentlinkops gsc-links --property PROPERTY --session SESSION --out FILE.csv [--kind latest|sample]
+
+Export GSC links through your signed-in browser and installed Playwriter.
+Use an existing Playwriter session. The command opens and closes its own tab.
+Files must be new. CSV exports are limited to 20 MiB and include a receipt.
+Review column mapping before importing. The export does not prove links are live.`;
+
+// This factory also runs in Playwriter's CommonJS sandbox. Keep dependencies explicit.
+export function createGscBrowserExporter({ fs, path, createHash, parseCsv }) {
+  const limit = 20 * 1024 * 1024;
+  const fail = (code, message) => Object.assign(new Error(message), { code, exitCode: 2 });
+  async function bounded(promise, milliseconds) {
+    let timer;
+    try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(fail('download_timeout', 'The CSV download timed out. Retry the export.')), milliseconds); })]); }
+    finally { clearTimeout(timer); }
+  }
+  function propertyValue(value) {
+    if (typeof value !== 'string' || value.length > 2048 || /[\u0000-\u0020\u007f]/u.test(value)) throw fail('invalid_property', 'Provide an exact GSC property.');
+    if (value.startsWith('sc-domain:')) {
+      const host = value.slice(10);
+      if (host.length > 253 || !/^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z0-9-]*[a-z0-9])?$/i.test(host) || host.split('.').some(x => x.length > 63)) throw fail('invalid_property', 'Provide a valid GSC domain property.');
+      return value;
+    }
+    let url;
+    try { url = new URL(value); } catch { throw fail('invalid_property', 'Provide a valid GSC URL property.'); }
+    if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash || url.href !== value) throw fail('invalid_property', 'Use the exact GSC URL property, including its trailing slash.');
+    return value;
+  }
+  function assertPage(page, property, { initial = false } = {}) {
+    const current = page.url();
+    if (initial && current === 'about:blank') return;
+    let url;
+    try { url = new URL(current); } catch { throw fail('unexpected_origin', 'The browser left Search Console.'); }
+    if (url.origin === 'https://accounts.google.com') throw fail('login_required', 'Sign in to Google in your browser, then retry.');
+    if (url.origin !== 'https://search.google.com' || url.username || url.password) throw fail('unexpected_origin', 'The browser left Search Console.');
+    if (url.pathname !== '/search-console/links' || url.searchParams.getAll('resource_id').length !== 1 || url.searchParams.get('resource_id') !== property) throw fail('property_mismatch', 'The browser property does not match the requested GSC property.');
+  }
+  async function pageState(page, property) {
+    assertPage(page, property);
+    const body = await page.locator('body').innerText({ timeout: 5000 });
+    if (/processing data|check (?:back )?again in (?:a (?:day|few days)|a day or so)/i.test(body)) throw fail('processing_data', 'Google is processing this property. Retry when its Links report has data.');
+    if (/you (?:do not|don.t) have (?:access|permission)|access denied|verify (?:your )?ownership|ownership verification/i.test(body)) throw fail('permission_required', 'This Google account needs access to the selected property.');
+    if (/sign in to (?:continue|google search console)/i.test(body)) throw fail('login_required', 'Sign in to Google in your browser, then retry.');
+  }
+  async function visibleChoice(page, text) {
+    for (const locator of [page.getByRole('menuitem', { name: text, exact: true }), page.getByRole('button', { name: text, exact: true }), page.getByText(text, { exact: true })]) {
+      if (await locator.count() === 1 && await locator.isVisible()) return locator;
+    }
+    return null;
+  }
+  async function choice(page, property, text, timeout = 10000) {
+    const until = Date.now() + timeout;
+    do {
+      await pageState(page, property);
+      const found = await visibleChoice(page, text);
+      if (found) return found;
+      await new Promise(resolve => setTimeout(resolve, 200));
+    } while (Date.now() < until);
+    throw fail('export_unavailable', 'The expected GSC export control is unavailable. Open the Links report to check its state.');
+  }
+  return async function exportLinks({ page, property, outputPath, kind = 'latest', now = () => new Date() }) {
+    property = propertyValue(property);
+    if (!['latest', 'sample'].includes(kind)) throw fail('invalid_kind', 'Choose latest or sample.');
+    if (typeof outputPath !== 'string' || !path.isAbsolute(outputPath) || !outputPath.toLowerCase().endsWith('.csv') || /[\u0000-\u001f\u007f]/u.test(outputPath)) throw fail('invalid_output', 'Choose an absolute CSV output path.');
+    const parent = await fs.realpath(path.dirname(outputPath));
+    if (parent !== path.resolve(path.dirname(outputPath))) throw fail('unsafe_output', 'The output directory must not use symbolic links.');
+    const receiptPath = `${outputPath}.receipt.json`;
+    for (const candidate of [outputPath, receiptPath]) {
+      try { await fs.lstat(candidate); throw fail('output_exists', 'The output or receipt file already exists. Choose a new path.'); }
+      catch (error) { if (error.code !== 'ENOENT') throw error; }
+    }
+    assertPage(page, property, { initial: true });
+    const report = new URL('https://search.google.com/search-console/links');
+    report.searchParams.set('resource_id', property);
+    report.searchParams.set('hl', 'en');
+    let download, ownedOutput = false, ownedReceipt = false;
+    const receive = item => { if (!download) download = item; };
+    try {
+      await page.goto(report.href, { waitUntil: 'domcontentloaded', timeout: 20000 });
+      const exportButton = await choice(page, property, 'Export external links');
+      await exportButton.click({ timeout: 5000 });
+      await pageState(page, property);
+      const kindButton = await choice(page, property, kind === 'latest' ? 'Latest links' : 'More sample links');
+      page.on('download', receive);
+      await kindButton.click({ timeout: 5000 });
+      const until = Date.now() + 15000;
+      let csvClicked = false;
+      do {
+        await pageState(page, property);
+        if (download) break;
+        if (!csvClicked) {
+          const csvButton = await visibleChoice(page, 'Download CSV') || await visibleChoice(page, 'CSV');
+          if (csvButton) { await csvButton.click({ timeout: 5000 }); csvClicked = true; }
+        }
+        if (!download) await new Promise(resolve => setTimeout(resolve, 200));
+      } while (!download && Date.now() < until);
+      if (!download) throw fail('download_unavailable', 'Google did not provide a CSV download. Check the Links report and retry.');
+      assertPage(page, property);
+      // The browser owns authenticated retrieval. Never extract cookies or replay private RPCs.
+      let stream;
+      try { stream = await bounded(download.createReadStream(), 15000); } catch { /* The extension can emit downloads without a relay-side artifact. */ }
+      const hash = createHash('sha256');
+      const chunks = [];
+      let bytes = 0;
+      async function accept(data) {
+        const chunk = Buffer.from(data);
+        bytes += chunk.length;
+        if (bytes > limit) throw fail('export_too_large', 'The CSV export exceeds the 20 MiB limit.');
+        hash.update(chunk); chunks.push(chunk);
+      }
+      let retrievalTransport = 'download-stream';
+      try {
+        if (stream && typeof stream[Symbol.asyncIterator] === 'function') {
+          const iterator = stream[Symbol.asyncIterator]();
+          const streamDeadline = Date.now() + 15000;
+          while (true) {
+            const remaining = streamDeadline - Date.now();
+            if (remaining <= 0) throw fail('download_timeout', 'The CSV download timed out. Retry the export.');
+            const {value: data, done} = await bounded(iterator.next(), remaining);
+            if (done) break;
+            await accept(data);
+          }
+        }
+      } finally { stream?.destroy?.(); }
+      if (!bytes) {
+        // Extension mode may have no relay-side file. Read only the URL emitted by
+        // this page's download event, inside that same signed-in page. No cookies,
+        // request headers or private RPC identifiers leave the browser.
+        const downloadUrl = download.url();
+        const parsed = new URL(downloadUrl);
+        const allowed = (parsed.protocol === 'https:' && parsed.origin === 'https://search.google.com' && !parsed.username && !parsed.password) ||
+          (parsed.protocol === 'blob:' && parsed.origin === 'https://search.google.com') ||
+          /^data:text\/csv[;,]/i.test(downloadUrl);
+        if (!allowed) throw fail('unsafe_download', 'Google returned an unsupported download address. Export the CSV manually.');
+        assertPage(page, property);
+        const result = await bounded(page.evaluate(async ({ url, maxBytes }) => {
+          const controller = new AbortController();
+          const timer = setTimeout(() => controller.abort(), 12000);
+          let reader;
+          try {
+            const response = await fetch(url, { credentials: 'include', redirect: 'error', signal: controller.signal });
+            if (!response.ok || !response.body) return { error: 'download_unavailable' };
+            if (Number(response.headers.get('content-length')) > maxBytes) return { error: 'export_too_large' };
+            reader = response.body.getReader();
+            let total = 0;
+            const encoded = [];
+            while (true) {
+              const { value, done } = await reader.read();
+              if (done) break;
+              total += value.byteLength;
+              if (total > maxBytes) return { error: 'export_too_large' };
+              let binary = '';
+              for (let offset = 0; offset < value.length; offset += 32768) binary += String.fromCharCode(...value.subarray(offset, offset + 32768));
+              encoded.push(btoa(binary));
+            }
+            return { chunks: encoded };
+          } catch { return { error: 'download_unavailable' }; }
+          finally { clearTimeout(timer); await reader?.cancel().catch(() => {}); controller.abort(); }
+        }, { url: downloadUrl, maxBytes: limit }), 15000);
+        if (result.error) throw fail(result.error, result.error === 'export_too_large' ? 'The CSV export exceeds the 20 MiB limit.' : 'The browser could not read its CSV download. Export it manually and import the file.');
+        for (const encoded of result.chunks) await accept(Buffer.from(encoded, 'base64'));
+        retrievalTransport = 'browser-download-url';
+      }
+      const csv = Buffer.concat(chunks).toString('utf8');
+      if (!bytes || /^\s*</.test(csv) || csv.includes('\u0000')) throw fail('invalid_csv', 'The downloaded file is not a CSV export.');
+      let rows;
+      try { rows = parseCsv(csv); } catch { throw fail('invalid_csv', 'The CSV export has invalid quoting.'); }
+      const headers = rows[0]?.map(value => value.trim()) ?? [];
+      const sourceHeader = headers.find(value => /^(linking page|links|source url)$/i.test(value));
+      if (!sourceHeader || headers.length > 30 || new Set(headers).size !== headers.length || headers.some(value => /^(top linking sites|top linked pages|domain|link count|external links)$/i.test(value)) || rows.slice(1).some(row => row.length !== headers.length)) throw fail('unsupported_csv', 'The CSV columns differ from the supported GSC exports. Review the export format.');
+      const sourceIndex = headers.indexOf(sourceHeader);
+      if (rows.slice(1).some(row => {
+        try { const source = new URL(row[sourceIndex]); return !['http:', 'https:'].includes(source.protocol) || Boolean(source.username || source.password); }
+        catch { return true; }
+      })) throw fail('unsupported_csv', 'The source column must contain page URLs. Review the export format.');
+      const timestamp = typeof now === 'function' ? now() : now;
+      const receipt = {
+        schema_version: 1, retrieved_by: 'gsc-browser-export', retrieval_transport: retrievalTransport, property, kind,
+        captured_at: new Date(timestamp).toISOString(), output_path: outputPath,
+        sha256: hash.digest('hex'), bytes, rows: rows.length - 1, headers,
+        source_column: sourceHeader, target_mapping_required: true,
+        sampled: true, current_link_status: 'unverified',
+      };
+      await fs.writeFile(outputPath, Buffer.concat(chunks), { flag: 'wx', mode: 0o600 }); ownedOutput = true;
+      await fs.writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { flag: 'wx', mode: 0o600 }); ownedReceipt = true;
+      return receipt;
+    } catch (error) {
+      await download?.cancel?.().catch(() => {});
+      if (ownedOutput) await fs.unlink(outputPath).catch(() => {});
+      if (ownedReceipt) await fs.unlink(receiptPath).catch(() => {});
+      if (error.exitCode === 2) throw error;
+      throw fail('browser_export_failed', 'GSC export failed. Check your browser session and retry.');
+    } finally {
+      page.off('download', receive);
+      await download?.delete?.().catch(() => {});
+    }
+  };
+}
+
+export const exportGscLinks = createGscBrowserExporter({ fs, path, createHash, parseCsv });
+
+export async function gscLinksMain(argv, { cwd = process.cwd(), out = console.log, execFileImpl = promisify(execFile) } = {}) {
+  const args = parseArgs(argv);
+  if (args._[0] === 'gsc-links') args._.shift();
+  if (args.help === true && Object.keys(args).length === 2 && !args._.length) { out(GSC_BROWSER_HELP); return 0; }
+  if (args._.length || Object.keys(args).some(key => !['_', 'property', 'session', 'out', 'kind'].includes(key)) ||
+      ['property', 'session', 'out'].some(key => typeof args[key] !== 'string' || !args[key]) ||
+      !/^[a-zA-Z0-9_-]{1,100}$/.test(args.session) || (args.kind !== undefined && !['latest', 'sample'].includes(args.kind))) {
+    throw Object.assign(new Error(GSC_BROWSER_HELP), { exitCode: 2 });
+  }
+  const outputPath = path.resolve(cwd, args.out);
+  const scratch = await fs.mkdtemp(path.join(tmpdir(), 'agentlinkops-gsc-'));
+  try {
+    // Pass source/config as a file argument, never interpolate a shell command.
+    const script = `const CsvError = ${CsvError.toString()};\nconst parseCsv = ${parseCsv.toString()};\nconst exportLinks = (${createGscBrowserExporter.toString()})({fs:require('node:fs').promises,path:require('node:path'),createHash:require('node:crypto').createHash,parseCsv});\nconst config = ${JSON.stringify({ property: args.property, outputPath, kind: args.kind ?? 'latest' })};\nconst exportPage = await context.newPage();\ntry { const receipt = await exportLinks({...config,page:exportPage}); console.log('AGENTLINKOPS_GSC_RECEIPT '+JSON.stringify(receipt)); } catch(error) { console.log('AGENTLINKOPS_GSC_ERROR '+JSON.stringify({code:error.code||'browser_export_failed',message:error.message})); } finally { await exportPage.close(); }\n`;
+    const scriptPath = path.join(scratch, 'export.js');
+    await fs.writeFile(scriptPath, script, { mode: 0o600 });
+    let stdout;
+    try { ({ stdout } = await execFileImpl('playwriter', ['-s', args.session, '-f', scriptPath, '--timeout', '60000'], { cwd, timeout: 65000, maxBuffer: 1024 * 1024 })); }
+    catch { throw Object.assign(new Error('Playwriter could not finish the export. Check the session and installed command.'), { code: 'playwriter_failed', exitCode: 2 }); }
+    const lines = stdout.split('\n').map(line => line.replace(/^\[log\]\s*/, ''));
+    const errorLine = lines.find(line => line.startsWith('AGENTLINKOPS_GSC_ERROR '));
+    if (errorLine) { const error = JSON.parse(errorLine.slice('AGENTLINKOPS_GSC_ERROR '.length)); throw Object.assign(new Error(error.message), { code: error.code, exitCode: 2 }); }
+    const receiptLine = lines.find(line => line.startsWith('AGENTLINKOPS_GSC_RECEIPT '));
+    if (!receiptLine) throw Object.assign(new Error('Playwriter returned no export receipt.'), { code: 'receipt_missing', exitCode: 2 });
+    const receipt = JSON.parse(receiptLine.slice('AGENTLINKOPS_GSC_RECEIPT '.length));
+    out(JSON.stringify(receipt, null, 2));
+    return 0;
+  } finally { await fs.rm(scratch, { recursive: true, force: true }); }
+}

package/cli/import.js

@@ -0,0 +1,108 @@
+// `agentlinkops import`: a supplier export becomes candidate rows in the shared shape.
+//
+// The whole lane exists so that "we do not make you depend on our data" is true today rather
+// than after the corpus is built. The customer keeps their supplier; we verify and watch.
+import { readFile } from 'node:fs/promises';
+import { readTable, CsvError } from './adapters/csv.js';
+import { resolveMapping, toImportRow, SUPPLIER_NAMES } from './adapters/suppliers.js';
+import { prepareImport } from '../src/discovery/import.js';
+import { DISCOVERY_LIMITS } from '../src/discovery/contract.js';
+
+/** Rejection reasons by frequency, with an example line for each. */
+export function summarizeReasons(rejected) {
+  const groups = new Map();
+  for (const entry of rejected) {
+    // "source_url: invalid_url" and "source_url: empty" are different problems; the row number
+    // inside a reason is not part of the kind.
+    const kind = String(entry.reason).replace(/\(.*?\)/gu, '(…)').slice(0, 120);
+    if (!groups.has(kind)) groups.set(kind, { reason: kind, count: 0, example_row: entry.row });
+    groups.get(kind).count++;
+  }
+  return [...groups.values()].sort((a, b) => b.count - a.count);
+}
+
+/** `--map source=Referring page URL,target=Target URL` */
+export function parseMap(value) {
+  if (!value || value === true) return {};
+  const overrides = {};
+  for (const pair of String(value).split(',')) {
+    const index = pair.indexOf('=');
+    if (index < 1) throw new CsvError(`--map entries look like field=column, not "${pair}"`, 0);
+    overrides[pair.slice(0, index).trim()] = pair.slice(index + 1).trim();
+  }
+  return overrides;
+}
+
+/**
+ * Reads an export and reports what it would import. Never writes.
+ *
+ * Rejections are returned in two layers because they have different causes: a row the FILE could
+ * not produce (a short row, a broken delimiter) and a row the CONTRACT would not accept (a bad
+ * URL, an unreadable date, a link to someone else's site). Collapsing them into one count would
+ * hide which half of the pipeline to look at.
+ */
+export async function readImport(path, {
+  supplier = 'csv', map = {}, target, targetKind = 'domain', includeSubdomains = true,
+  runId = 'dr_preview', workspaceId = 'ws_preview', projectId = 'pr_preview', now = new Date().toISOString(),
+  supplierGeneratedAt = null,
+} = {}) {
+  if (!SUPPLIER_NAMES.includes(supplier)) return { error: `unknown supplier "${supplier}". Known: ${SUPPLIER_NAMES.join(', ')}` };
+  const text = await readFile(path, 'utf8');
+  let table;
+  try { table = readTable(text); }
+  catch (error) { return { error: error instanceof CsvError ? `line ${error.line}: ${error.message}` : String(error.message) }; }
+
+  const resolved = resolveMapping(table.header, { supplier, overrides: map, targetProvided: Boolean(target) });
+  if (resolved.error) return { error: resolved.error, header: table.header };
+
+  const unreadable = table.records.filter(record => record.error).map(record => ({ row: record.line, reason: record.error }));
+  const readable = table.records.filter(record => record.values);
+  const rows = readable.map(record => toImportRow(record.values, resolved.mapping, resolved.unclaimed));
+  // A file with no target column is a file about ONE target, which the operator named with
+  // --target. Every row gets that target's site root; the scope below says what "matching"
+  // means. Nothing is inferred — without --target the resolver above refuses the file.
+  if (!resolved.mapping.target) {
+    const targetUrl = targetKind === 'domain' ? `https://${target}/` : target;
+    for (const row of rows) row.target_url = targetUrl;
+  }
+
+  const targetUrl = targetKind === 'domain' ? `https://${target}/` : target;
+  const scope = targetKind === 'exact_url' ? 'exact' : includeSubdomains ? 'domain' : 'subdomain';
+  // One discovery run holds at most 1,000 candidates, and real exports are bigger than that —
+  // the Settled registry is 2,241 rows. A preview reads the WHOLE file and reports it as one
+  // answer; `sync` is where the chunks become runs. Duplicate detection is shared across
+  // chunks, so a placement repeated in two different parts of one file is still one candidate.
+  const prepared = { accepted: [], rejected: [], duplicates: [] };
+  const seen = new Map();
+  const chunks = [];
+  for (let index = 0; index < rows.length; index += DISCOVERY_LIMITS.rows) chunks.push(index);
+  for (const offset of chunks) {
+    const part = await prepareImport({
+      runId: `${runId}_${offset / DISCOVERY_LIMITS.rows}`, workspaceId, projectId, supplier,
+      rows: rows.slice(offset, offset + DISCOVERY_LIMITS.rows), target: targetUrl, scope,
+      taskId: 'import_preview', retrievedAt: now, generatedAt: supplierGeneratedAt,
+      seen, rowOffset: offset,
+    });
+    prepared.accepted.push(...part.accepted);
+    prepared.rejected.push(...part.rejected);
+    prepared.duplicates.push(...part.duplicates);
+  }
+  // Rejections carry the file's own line numbers, not the array index, so "row 412" means what
+  // it means when the customer opens the file.
+  const rejected = [
+    ...unreadable,
+    ...prepared.rejected.map(entry => ({ row: readable[entry.row - 1]?.line ?? entry.row, reason: entry.reason })),
+  ].sort((a, b) => a.row - b.row);
+
+  return {
+    supplier, mapping: resolved.mapping, unclaimed: resolved.unclaimed, header: table.header,
+    rows, accepted: prepared.accepted,
+    rejected, duplicates: prepared.duplicates.map(entry => ({ row: readable[entry.row - 1]?.line ?? entry.row, first_seen_on_row: readable[entry.first_seen_on_row - 1]?.line ?? entry.first_seen_on_row })),
+    counts: { read: table.records.length, accepted: prepared.accepted.length, rejected: rejected.length, duplicates: prepared.duplicates.length },
+    chunks: chunks.length,
+    // Grouped, because two thousand identical lines is not a report. When one reason accounts
+    // for nearly everything, the MAPPING is wrong rather than the data, and the reader needs to
+    // see that in the first line instead of inferring it from scrolling.
+    reasons: summarizeReasons(rejected),
+  };
+}

package/cli/ledger.js

@@ -0,0 +1,232 @@
+// The ledger: intent, human-owned, one JSON object per line.
+//
+// Reading is deliberately forgiving of everything except silence. A line that does not parse,
+// or an entry that is not valid, is REPORTED with its line number and kept out of the working
+// set — never dropped quietly. A tool that discards a customer's row and carries on produces a
+// number that is wrong in a way they discover months later.
+import { readFile, writeFile, mkdir, rename, open, unlink } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+export const INTENTS = Object.freeze(['wanted', 'expected', 'retired']);
+export const SCOPES = Object.freeze(['exact', 'domain', 'subdomain', 'path']);
+export const CADENCES = Object.freeze({ daily: 86_400, weekly: 604_800, monthly: 2_592_000 });
+// What the cloud accepts, mirrored so a local ledger cannot hold something sync must reject.
+export const CADENCE_BOUNDS = Object.freeze({ min: 3_600, max: 2_592_000 });
+// Which transition is news, per intent. This is the whole difference between them.
+export const DEFAULT_CADENCE = Object.freeze({ wanted: 'weekly', expected: 'daily', retired: null });
+
+// Stable key order, so `fmt` produces a diff that names the entry that changed rather than
+// reordering every line.
+const KEY_ORDER = ['id', 'intent', 'source', 'target', 'scope', 'expect', 'cadence', 'ref', 'origin', 'tags', 'added', 'note'];
+
+// Where an entry came from, when it was not typed by hand. `placement_run` is adopt-side
+// lineage, not import-side: a run's queue holds decisions already made and its output reports
+// placements, which is why T09 consumes it through adopt rather than the discovery import lane.
+export const ORIGIN_KINDS = Object.freeze(['manual', 'import', 'discovery_candidate', 'mention', 'citation', 'competitor', 'placement_run']);
+// What `local_reference` can carry to the cloud. It truncates at 200, and the ledger id plus a
+// space goes first — so how much `ref` survives depends on the ID, which is 7 to 35 characters.
+// A single constant would be wrong for every entry but one, which is why the check below is a
+// function of the entry rather than a number.
+export const LOCAL_REFERENCE_LIMIT = 200;
+
+export function newId() {
+  return `lk_${crypto.randomUUID().replaceAll('-', '').slice(0, 10)}`;
+}
+
+const isUrl = value => {
+  try { const url = new URL(value); return url.protocol === 'https:' || url.protocol === 'http:'; }
+  catch { return false; }
+};
+
+/** Validates one entry and returns it normalised, or throws with a reason a human can act on. */
+export function normalizeEntry(input, { assignId = false } = {}) {
+  const problems = [];
+  const entry = {};
+  entry.id = typeof input.id === 'string' && /^lk_[a-z0-9]{4,32}$/.test(input.id)
+    ? input.id : (assignId ? newId() : null);
+  if (!entry.id) problems.push('id must look like lk_xxxxxxxx');
+  entry.intent = INTENTS.includes(input.intent) ? input.intent : null;
+  if (!entry.intent) problems.push(`intent must be one of ${INTENTS.join(', ')}`);
+  entry.source = isUrl(input.source) ? input.source : null;
+  if (!entry.source) problems.push('source must be an http(s) URL');
+  entry.target = isUrl(input.target) ? input.target : null;
+  if (!entry.target) problems.push('target must be an http(s) URL');
+  entry.scope = input.scope ?? 'exact';
+  if (!SCOPES.includes(entry.scope)) problems.push(`scope must be one of ${SCOPES.join(', ')}`);
+
+  if (input.expect != null) {
+    if (typeof input.expect !== 'object' || Array.isArray(input.expect)) problems.push('expect must be an object');
+    else {
+      const expect = {};
+      if (input.expect.anchor != null) {
+        if (typeof input.expect.anchor !== 'string' || input.expect.anchor.length > 1000) problems.push('expect.anchor must be a string of at most 1000 characters');
+        else expect.anchor = input.expect.anchor;
+      }
+      if (input.expect.rel != null) {
+        const rel = input.expect.rel;
+        if (!Array.isArray(rel) || rel.length > 20 || rel.some(token => typeof token !== 'string' || !/^[a-zA-Z0-9_-]{1,40}$/.test(token))) {
+          problems.push('expect.rel must be up to 20 relation tokens');
+        } else expect.rel = [...new Set(rel.map(token => token.toLowerCase()))].sort();
+      }
+      if (Object.keys(expect).length) entry.expect = expect;
+    }
+  }
+
+  const cadence = input.cadence ?? DEFAULT_CADENCE[entry.intent] ?? undefined;
+  if (cadence !== undefined && cadence !== null) {
+    const seconds = cadenceSeconds(cadence);
+    if (seconds === null) problems.push(`cadence must be daily, weekly, monthly or ${CADENCE_BOUNDS.min}-${CADENCE_BOUNDS.max} seconds`);
+    else entry.cadence = cadence;
+  }
+  if (input.ref != null) {
+    if (typeof input.ref !== 'string' || input.ref.length > 500) problems.push('ref must be a string of at most 500 characters');
+    else entry.ref = input.ref;
+  }
+  // Lineage: where this entry came from, when a tool produced it rather than a person typing it.
+  //
+  // **Deliberately its own field and not part of `ref`.** `ref` is the half of `local_reference`
+  // the cloud truncates at 200 characters, and how much of it survives depends on the length of
+  // the ledger id in front of it — so the same provenance string survives on one entry and is cut
+  // on another. A provenance that shortens depending on an unrelated id is one that will name the
+  // wrong run, so lineage stays local, where nothing truncates it.
+  if (input.origin != null) {
+    if (typeof input.origin !== 'object' || Array.isArray(input.origin)) problems.push('origin must be an object');
+    else {
+      const origin = {};
+      if (!ORIGIN_KINDS.includes(input.origin.kind)) problems.push(`origin.kind must be one of ${ORIGIN_KINDS.join(', ')}`);
+      else origin.kind = input.origin.kind;
+      for (const field of ['id', 'run_id', 'provider', 'observed_at', 'source_url']) {
+        if (input.origin[field] == null) continue;
+        if (typeof input.origin[field] !== 'string' || input.origin[field].length > 500) problems.push(`origin.${field} must be a string of at most 500 characters`);
+        else origin[field] = input.origin[field];
+      }
+      // An unknown lineage key is kept for the same reason an unknown entry key is.
+      for (const [key, value] of Object.entries(input.origin)) if (!(key in origin) && !['kind', 'id', 'run_id', 'provider', 'observed_at', 'source_url'].includes(key)) origin[key] = value;
+      if (Object.keys(origin).length) entry.origin = origin;
+    }
+  }
+  if (input.tags != null) {
+    if (!Array.isArray(input.tags) || input.tags.some(tag => typeof tag !== 'string')) problems.push('tags must be an array of strings');
+    else if (input.tags.length) entry.tags = [...new Set(input.tags)];
+  }
+  for (const field of ['added', 'note']) {
+    if (input[field] != null) {
+      if (typeof input[field] !== 'string') problems.push(`${field} must be a string`);
+      else entry[field] = input[field];
+    }
+  }
+  // An unknown key is kept rather than discarded: it is the customer's file, and a future
+  // version of this tool may well mean something by it.
+  for (const [key, value] of Object.entries(input)) if (!KEY_ORDER.includes(key)) entry[key] = value;
+  if (problems.length) {
+    const error = new Error(problems.join('; '));
+    error.problems = problems;
+    throw error;
+  }
+  return entry;
+}
+
+/** `daily` / `weekly` / `monthly` / an integer, to seconds the cloud will accept. */
+export function cadenceSeconds(cadence) {
+  const seconds = typeof cadence === 'string' ? (CADENCES[cadence] ?? Number(cadence)) : cadence;
+  if (!Number.isInteger(seconds) || seconds < CADENCE_BOUNDS.min || seconds > CADENCE_BOUNDS.max) return null;
+  return seconds;
+}
+
+/**
+ * Reads a ledger. Returns the entries that are usable AND every problem found, because a
+ * caller that cannot see the problems will report a count that quietly excludes them.
+ */
+export async function readLedger(path) {
+  let text = '';
+  try { text = await readFile(path, 'utf8'); }
+  catch (error) { if (error.code === 'ENOENT') return { entries: [], problems: [], missing: true }; throw error; }
+  const entries = [], problems = [], seen = new Map();
+  text.split('\n').forEach((line, index) => {
+    const number = index + 1;
+    if (!line.trim() || line.trimStart().startsWith('//')) return;
+    let parsed;
+    try { parsed = JSON.parse(line); }
+    catch (error) { problems.push({ line: number, reason: `not valid JSON: ${error.message}` }); return; }
+    let entry;
+    try { entry = normalizeEntry(parsed); }
+    catch (error) { problems.push({ line: number, id: parsed?.id ?? null, reason: error.message }); return; }
+    // A duplicate id is an ambiguity, not a preference: which observation history belongs to
+    // which entry stops having an answer, so both are reported and neither is guessed at.
+    if (seen.has(entry.id)) {
+      problems.push({ line: number, id: entry.id, reason: `duplicate id, first seen on line ${seen.get(entry.id)}` });
+      return;
+    }
+    seen.set(entry.id, number);
+    entries.push(entry);
+  });
+  return { entries, problems, missing: false };
+}
+
+/** One entry per line, keys in a fixed order, sorted by id. Written atomically. */
+export function serializeLedger(entries) {
+  const ordered = [...entries].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return ordered.map(entry => {
+    const object = {};
+    for (const key of KEY_ORDER) if (entry[key] !== undefined) object[key] = entry[key];
+    for (const [key, value] of Object.entries(entry)) if (!KEY_ORDER.includes(key)) object[key] = value;
+    return JSON.stringify(object);
+  }).join('\n') + (ordered.length ? '\n' : '');
+}
+
+export async function writeLedger(path, entries) {
+  await mkdir(dirname(path), { recursive: true });
+  const temporary = join(dirname(path), `.${crypto.randomUUID()}.tmp`);
+  await writeFile(temporary, serializeLedger(entries), 'utf8');
+  await rename(temporary, path);
+}
+
+/** Read, validate and replace under one lock. A malformed row prevents every mutation. */
+export async function mutateLedger(path, update) {
+  const lockPath = `${path}.lock`;
+  const lock = await open(lockPath, 'wx').catch(error => {
+    if (error.code === 'EEXIST') throw new Error('Ledger is being updated by another process.');
+    throw error;
+  });
+  try {
+    const ledger = await readLedger(path);
+    if (ledger.missing) throw new Error('Ledger does not exist.');
+    if (ledger.problems.length) throw new Error(`Ledger has invalid rows; repair them before writing (line ${ledger.problems[0].line}).`);
+    const result = await update(ledger.entries);
+    await writeLedger(path, result);
+    return result;
+  } finally { await lock.close(); await unlink(lockPath); }
+}
+
+/**
+ * What the cloud is told about this entry.
+ *
+ * `localReference` carries the ledger id FIRST and the customer's own reference after it, so
+ * the id survives truncation at the cloud's 200-character limit. The id is what makes a
+ * corrected URL keep its history; the customer's reference is best-effort and the ledger keeps
+ * the full value either way.
+ */
+export function toWatchInput(entry) {
+  // The ledger id always survives; the ref tail may not, and `refTruncatedInTransit` exists so a
+  // caller can say so rather than discovering it when a reference comes back short.
+  const reference = entry.ref ? `${entry.id} ${entry.ref}` : entry.id;
+  return {
+    sourceUrl: entry.source, targetUrl: entry.target, targetScope: entry.scope,
+    cadenceSeconds: cadenceSeconds(entry.cadence ?? DEFAULT_CADENCE[entry.intent] ?? 'daily'),
+    expectedAnchor: entry.expect?.anchor ?? null,
+    expectedRel: entry.expect?.rel ?? null,
+    localReference: reference.slice(0, 200),
+  };
+}
+
+/** Whether this entry's `ref` would lose its tail on the way to the cloud. */
+export function refTruncatedInTransit(entry) {
+  if (typeof entry?.ref !== 'string' || !entry.ref) return false;
+  return `${entry.id} ${entry.ref}`.length > LOCAL_REFERENCE_LIMIT;
+}
+
+/** The ledger id inside a `localReference`, or null. */
+export function ledgerIdOf(localReference) {
+  const match = /^(lk_[a-z0-9]{4,32})(?:\s|$)/.exec(localReference ?? '');
+  return match ? match[1] : null;
+}

package/cli/linktrail.mjs

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+// Compatibility alias for the pilot window (DP-0029): the executable is `agentlinkops`.
+// One line to stderr, then the same main — stdout stays clean for `--json` readers.
+import { main } from './main.js';
+process.stderr.write('linktrail: this command is now `agentlinkops`; the old name keeps working during the pilot compatibility window.\n');
+process.exitCode = await main();