agentlinkops 0.5.0

package/src/competitors/contract.js

@@ -0,0 +1,157 @@
+import { z } from 'zod';
+import { DiscoveryQuery, DiscoveryRun, DiscoveryCandidate, candidateId, discoveryHash, queryHash, sourceQueryTerms } from '../discovery/contract.js';
+import { matchesTarget, validatePublicUrl } from '../verifier/url.js';
+
+// Pure internal boundary. No persistence, transport, provider calls or scheduling.
+export const COMPETITOR_VERSION = 1;
+export const COMPETITOR_LIMITS = Object.freeze({ competitors: 10, retrievalSkewMs: 86400000 });
+const identifier = z.string().regex(/^[a-zA-Z0-9_-]{1,128}$/);
+const timestamp = z.iso.datetime({ precision: 3 });
+// Scope validation only needs a well-formed query, so it borrows one source's ordering
+// terms. Snapshot comparison uses each run's own source, never this template.
+const policy = { backlinks_status_type: 'live', exclude_internal_backlinks: true, mode: 'as_is',
+  filters: null, ...sourceQueryTerms('dataforseo'), page_limit: 100, row_limit: 1000 };
+const fail = code => { throw new CompetitorContractError(code); };
+export class CompetitorContractError extends Error {
+  constructor(code) { super(code); this.code = code; }
+}
+
+export const TargetScope = z.strictObject({ target_kind: z.enum(['domain', 'exact_url']),
+  target: z.string().min(1).max(4096), include_subdomains: z.boolean(),
+}).superRefine((scope, ctx) => {
+  if (!DiscoveryQuery.safeParse({ ...scope, ...policy }).success)
+    ctx.addIssue({ code: 'custom', message: 'Expected an explicit canonical discovery target scope.' });
+});
+const memberSchema = z.strictObject({ id: identifier, role: z.enum(['customer', 'competitor']), scope: TargetScope });
+export const ApprovedCompetitorSet = z.strictObject({
+  v: z.literal(COMPETITOR_VERSION), id: identifier, revision: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER),
+  workspace_id: identifier, project_id: identifier, approved_by: identifier, approved_at: timestamp,
+  members: z.array(memberSchema).min(2).max(COMPETITOR_LIMITS.competitors + 1),
+}).superRefine((set, ctx) => {
+  if (set.members.filter(m => m.role === 'customer').length !== 1 || new Set(set.members.map(m => m.id)).size !== set.members.length)
+    ctx.addIssue({ code: 'custom', message: 'Exactly one customer and distinct member IDs are required.' });
+  if (set.members.some(m => !TargetScope.safeParse(m.scope).success)) return;
+  for (let i = 0; i < set.members.length; i++) for (let j = i + 1; j < set.members.length; j++) {
+    if (scopesOverlap(set.members[i].scope, set.members[j].scope))
+      ctx.addIssue({ code: 'custom', message: 'Approved target scopes must not overlap.' });
+  }
+});
+
+function contains(scope, url) {
+  return matchesTarget(url, scope.target_kind === 'domain' ? `https://${scope.target}/` : scope.target,
+    scope.target_kind === 'exact_url' ? 'exact' : scope.include_subdomains ? 'domain' : 'subdomain');
+}
+function scopesOverlap(a, b) {
+  if (a.target_kind === 'exact_url') return contains(b, a.target);
+  if (b.target_kind === 'exact_url') return contains(a, b.target);
+  return contains(a, `https://${b.target}/`) || contains(b, `https://${a.target}/`);
+}
+const freeze = value => {
+  if (value && typeof value === 'object') { Object.values(value).forEach(freeze); Object.freeze(value); }
+  return value;
+};
+function orderedSet(input) {
+  const set = ApprovedCompetitorSet.parse(input);
+  set.members.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+  return set;
+}
+export function approveCompetitorSet(input) {
+  // approved_by is supplied by the future authenticated service, never inferred
+  // from suggestions or from a provider's competitor ranking.
+  return freeze(orderedSet(input));
+}
+export async function competitorSetHash(input) { return discoveryHash(orderedSet(input)); }
+export async function targetScopeId(input) {
+  return `ts_${await discoveryHash([COMPETITOR_VERSION, TargetScope.parse(input)])}`;
+}
+export function sourceGroupKey(sourceUrl, grouping = 'page') {
+  const checked = validatePublicUrl(sourceUrl);
+  if (!checked.valid || checked.url !== sourceUrl || sourceUrl.length > 4096) fail('INVALID_SOURCE_URL');
+  if (grouping === 'page') return checked.url;
+  if (grouping === 'source_host') return checked.hostname;
+  fail('UNSUPPORTED_SOURCE_GROUPING');
+}
+
+const receiptSchema = z.strictObject({ provider_task_id: identifier, provider_retrieved_at: timestamp });
+const terminal = new Set(['succeeded', 'partial', 'failed', 'reconciliation_required']);
+
+export async function freezeInventorySnapshot({ id, set: inputSet, memberId, run: inputRun, candidates: inputCandidates, receipts: inputReceipts, capturedAt }) {
+  identifier.parse(id); timestamp.parse(capturedAt);
+  const set = orderedSet(inputSet), run = DiscoveryRun.parse(inputRun);
+  const member = set.members.find(m => m.id === memberId);
+  if (!member) fail('COMPETITOR_MEMBER_NOT_FOUND');
+  if (run.workspace_id !== set.workspace_id || run.project_id !== set.project_id) fail('SNAPSHOT_SCOPE_MISMATCH');
+  if (!terminal.has(run.status) || !run.finished_at) fail('SNAPSHOT_RUN_NOT_TERMINAL');
+  if (run.query_hash !== await queryHash(run.query) || await targetScopeId(member.scope) !== await targetScopeId(scopeFromQuery(run.query)))
+    fail('SNAPSHOT_QUERY_MISMATCH');
+  if (run.created_at < set.approved_at || (run.started_at !== null && run.started_at < run.created_at) ||
+      run.finished_at < (run.started_at ?? run.created_at) || capturedAt < run.finished_at) fail('SNAPSHOT_TIME_MISMATCH');
+  const candidates = z.array(DiscoveryCandidate).max(run.query.row_limit).parse(inputCandidates);
+  const receipts = z.array(receiptSchema).max(1000).parse(inputReceipts);
+  if (candidates.length !== run.usage.accepted_candidates || new Set(candidates.map(c => c.id)).size !== candidates.length)
+    fail('SNAPSHOT_INVENTORY_INCOMPLETE');
+  if (receipts.length > run.usage.request_count || new Set(receipts.map(r => r.provider_task_id)).size !== receipts.length)
+    fail('SNAPSHOT_RECEIPT_MISMATCH');
+  const receiptById = new Map(receipts.map(r => [r.provider_task_id, r]));
+  for (const receipt of receipts) {
+    if (!run.started_at || receipt.provider_retrieved_at < run.started_at || receipt.provider_retrieved_at > run.finished_at)
+      fail('SNAPSHOT_TIME_MISMATCH');
+  }
+  for (const c of candidates) {
+    if (c.workspace_id !== run.workspace_id || c.project_id !== run.project_id || c.discovery_run_id !== run.id ||
+        c.provider !== run.provider || c.data_mode !== run.data_mode || !contains(member.scope, c.target_url) ||
+        c.id !== await candidateId(run.id, c.source_url, c.target_url)) fail('SNAPSHOT_CANDIDATE_MISMATCH');
+    if (receiptById.get(c.provider_task_id)?.provider_retrieved_at !== c.provider_retrieved_at) fail('SNAPSHOT_RECEIPT_MISMATCH');
+  }
+  if (run.coverage === 'complete_for_query' && (run.provider_total_count !== run.usage.returned_rows ||
+      receipts.length === 0 || receipts.length !== run.usage.request_count || run.coverage_reason !== null))
+    fail('SNAPSHOT_COMPLETENESS_MISMATCH');
+  candidates.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+  receipts.sort((a, b) => a.provider_task_id < b.provider_task_id ? -1 : a.provider_task_id > b.provider_task_id ? 1 : 0);
+  const dates = receipts.map(r => r.provider_retrieved_at).sort();
+  const snapshot = { v: COMPETITOR_VERSION, id, workspace_id: set.workspace_id, project_id: set.project_id,
+    set_id: set.id, set_revision: set.revision, set_hash: await competitorSetHash(set),
+    member_id: member.id, member_role: member.role, target_scope_id: await targetScopeId(member.scope),
+    captured_at: capturedAt, provider_retrieved_from: dates[0] ?? null, provider_retrieved_to: dates.at(-1) ?? null,
+    run, receipts, candidates };
+  return freeze({ ...snapshot, content_hash: await discoveryHash(snapshot) });
+}
+
+const scopeFromQuery = q => ({ target_kind: q.target_kind, target: q.target, include_subdomains: q.include_subdomains });
+const comparisonProfile = q => { const { target, ...profile } = DiscoveryQuery.parse(q); return profile; };
+
+export async function validateInventorySnapshot(snapshot, set) {
+  if (!snapshot || typeof snapshot !== 'object') fail('INVALID_INVENTORY_SNAPSHOT');
+  const reconstructed = await freezeInventorySnapshot({ id: snapshot.id, set, memberId: snapshot.member_id,
+    run: snapshot.run, candidates: snapshot.candidates, receipts: snapshot.receipts, capturedAt: snapshot.captured_at });
+  // Compare all fields; a hash alone is not validation or authorization.
+  if (Object.keys(snapshot).length !== Object.keys(reconstructed).length ||
+      Object.keys(reconstructed).some(key => JSON.stringify(snapshot[key]) !== JSON.stringify(reconstructed[key])))
+    fail('SNAPSHOT_CONTENT_MISMATCH');
+  return reconstructed;
+}
+
+export async function compareInventoryCollection({ set, snapshots, maxRetrievalSkewMs = COMPETITOR_LIMITS.retrievalSkewMs }) {
+  if (!Number.isSafeInteger(maxRetrievalSkewMs) || maxRetrievalSkewMs < 0 || maxRetrievalSkewMs > COMPETITOR_LIMITS.retrievalSkewMs)
+    fail('INVALID_COMPARISON_WINDOW');
+  if (!Array.isArray(snapshots) || snapshots.length < 2 || snapshots.length > COMPETITOR_LIMITS.competitors + 1)
+    fail('INVALID_COMPARISON_INVENTORIES');
+  const validated = [];
+  for (const snapshot of snapshots) validated.push(await validateInventorySnapshot(snapshot, set));
+  if (new Set(validated.map(s => s.member_id)).size !== validated.length) fail('COMPARISON_REQUIRES_DISTINCT_MEMBERS');
+  const a = validated[0];
+  const reasons = [];
+  if (validated.some(b => a.run.provider !== b.run.provider)) reasons.push('provider_mismatch');
+  if (validated.some(b => a.run.data_mode !== b.run.data_mode)) reasons.push('data_mode_mismatch');
+  if (validated.some(b => JSON.stringify(comparisonProfile(a.run.query)) !== JSON.stringify(comparisonProfile(b.run.query)))) reasons.push('query_profile_mismatch');
+  if (reasons.length) return { snapshots: validated, comparison: freeze({ mode: 'incompatible', reasons, absence_claim: 'not_supported' }) };
+  if (validated.some(s => s.run.coverage !== 'complete_for_query')) reasons.push('incomplete_query_coverage');
+  const dates = validated.flatMap(s => [s.provider_retrieved_from, s.provider_retrieved_to]);
+  if (dates.some(d => d === null)) reasons.push('retrieval_time_unknown');
+  else if (Math.max(...dates.map(Date.parse)) - Math.min(...dates.map(Date.parse)) > maxRetrievalSkewMs) reasons.push('retrieval_window_exceeded');
+  return { snapshots: validated, comparison: freeze({ mode: reasons.length ? 'observed_overlap_only' : 'complete_query_datasets', reasons, absence_claim: 'not_supported' }) };
+}
+
+export async function compareInventorySnapshots({ set, left, right, maxRetrievalSkewMs }) {
+  return (await compareInventoryCollection({ set, snapshots: [left, right], maxRetrievalSkewMs })).comparison;
+}

package/src/competitors/domain-mix.js

@@ -0,0 +1,199 @@
+import { z } from 'zod';
+import { validateInventorySnapshot, compareInventoryCollection, COMPETITOR_LIMITS } from './contract.js';
+import { discoveryHash } from '../discovery/contract.js';
+
+// The referring-domain mix report (DP-0003-T08).
+//
+// Compare imported referring-domain classes across customer lanes and selected inventories.
+//
+// The honesty rules are DP-0003-T06's, reused rather than reinvented:
+//   - a rank is `rank_within_dataset`, never `rank`, and the metadata says the ranking scope;
+//   - a thing we did not measure is unavailable or unlabelled, never zero or guessed;
+//   - one dataset is not a comparison, and a comparison is null rather than an empty object.
+//
+// The one rule this report adds: CLASS LABELS COME FROM IMPORTED ROWS. A referring domain
+// no imported row ever labelled is counted `unlabelled`, and the share of a member's domains
+// our labels reach is reported as label coverage. Inferring "generic" from a domain's shape
+// would present a guess as an imported label.
+export const DOMAIN_MIX_CLASSES = Object.freeze(['generic', 'niche']);
+
+const order = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+const hostOf = url => { try { return new URL(url).hostname.replace(/^www\./u, ''); } catch { return null; } };
+
+export const domainMixOptions = {
+  limit: z.number().int().min(1).max(200).optional(),
+  maxRetrievalSkewMs: z.number().int().min(0).max(COMPETITOR_LIMITS.retrievalSkewMs).optional(),
+};
+
+/** A lane's rows are whatever imported rows already carry: a URL, a class label, maybe a date. */
+export const laneRow = z.object({
+  source_url: z.string().min(1),
+  class: z.string().min(1).nullable().optional(),
+  added: z.string().nullable().optional(),
+});
+export const laneInput = z.object({ lane: z.string().min(1), rows: z.array(laneRow).min(1) });
+
+const coverageOf = snapshot => snapshot?.run?.coverage ?? snapshot?.coverage ?? 'unknown';
+
+/**
+ * One side's referring-domain split. `labelFor` maps a host to its imported label or null;
+ * every domain it cannot label is counted `unlabelled`, and that count is a COVERAGE fact.
+ */
+function memberSplit(domains, labelFor, labels, limit) {
+  // domains: Map<host, Set<source_url>> — distinct referring pages per host, T06's counting unit.
+  const classes = { unlabelled: 0 };
+  for (const label of labels) classes[label] = 0;
+  const hosts = [];
+  for (const [host, pages] of domains) {
+    const label = labelFor(host);
+    classes[label ?? 'unlabelled']++;
+    hosts.push({ host, class: label ?? 'unlabelled', referring_pages: pages.size });
+  }
+  const labelled = hosts.length - classes.unlabelled;
+  hosts.sort((a, b) => b.referring_pages - a.referring_pages || order(a.host, b.host));
+  return {
+    referring_domains: hosts.length,
+    classes,
+    labelled_domains: labelled,
+    label_coverage: hosts.length ? Number((labelled / hosts.length).toFixed(4)) : null,
+    // Ratios are over LABELLED domains only: an unlabelled domain has no class to be in, and a
+    // denominator that included it would read unlabelled as niche-by-default.
+    generic_share_of_labelled: labelled ? Number(((classes.generic ?? 0) / labelled).toFixed(4)) : null,
+    niche_share_of_labelled: labelled ? Number(((classes.niche ?? 0) / labelled).toFixed(4)) : null,
+    generic_niche_ratio: (classes.niche ?? 0) ? Number(((classes.generic ?? 0) / classes.niche).toFixed(4)) : null,
+    top_domains: hosts.slice(0, limit).map((row, index) => ({ ...row, rank_within_dataset: index + 1 })),
+    // The diff needs every domain, not the capped display list; compact on purpose.
+    domain_classes: Object.fromEntries(hosts.map(row => [row.host, row.class])),
+  };
+}
+
+/**
+ * The report. `lanes` is one or more labelled lanes (an adopted registry, an imported
+ * supplier file); `snapshots` are frozen competitor inventories, and MAY be empty — the
+ * report then renders ours alone and states the competitor side is unavailable rather than
+ * inventing comparison numbers.
+ */
+export async function buildDomainMixReport({ set = null, lanes, snapshots = [], ...input }) {
+  const options = z.strictObject(domainMixOptions).parse(input);
+  const limit = options.limit ?? 25;
+  const skew = options.maxRetrievalSkewMs ?? COMPETITOR_LIMITS.retrievalSkewMs;
+  const parsedLanes = lanes.map(lane => laneInput.parse(lane));
+  if (!parsedLanes.length) throw new Error('DOMAIN_MIX_REQUIRES_A_LANE');
+  // Ours alone is a legitimate request (the real lane today: no approved competitor set
+  // exists, so there is nothing to validate against). A set becomes required the moment a
+  // snapshot is selected, because a frozen inventory only means anything inside one.
+  if (snapshots.length && !set) throw new Error('DOMAIN_MIX_REQUIRES_A_SET_FOR_INVENTORIES');
+
+  // The label catalogue every side is classified against: host -> label, from our rows only.
+  const labelByHost = new Map();
+  const seenLabels = new Set();
+  for (const lane of parsedLanes) {
+    for (const row of lane.rows) {
+      const host = hostOf(row.source_url);
+      if (!host) continue;
+      if (row.class) { labelByHost.set(host, row.class); seenLabels.add(row.class); }
+      else if (!labelByHost.has(host)) labelByHost.set(host, null);
+    }
+  }
+  const labels = [...new Set([...DOMAIN_MIX_CLASSES, ...seenLabels])].filter(l => l !== 'unlabelled').sort();
+  const labelFor = host => labelByHost.get(host) ?? null;
+
+  const ours = parsedLanes.map(lane => {
+    const domains = new Map();
+    for (const row of lane.rows) {
+      const host = hostOf(row.source_url);
+      if (!host) continue;
+      if (!domains.has(host)) domains.set(host, new Set());
+      domains.get(host).add(row.source_url);
+    }
+    return { lane: lane.lane, rows_read: lane.rows.length, ...memberSplit(domains, labelFor, labels, limit) };
+  });
+
+  // Competitors: validated snapshots, each split against the same label catalogue. An empty
+  // selection is a legitimate request — ours alone — and is stated, not filled with anything.
+  const validated = [];
+  for (const snapshot of snapshots) validated.push(await validateInventorySnapshot(snapshot, set));
+  // A collection comparison is offered only when the datasets support one (T06's rule); with
+  // zero or one dataset it is null rather than an empty object, because empty reads as a result.
+  let comparison = null;
+  if (validated.length > 1) comparison = (await compareInventoryCollection({ set, snapshots: validated, maxRetrievalSkewMs: skew })).comparison;
+  const members = [];
+  for (const snapshot of validated) {
+    const domains = new Map();
+    for (const candidate of snapshot.candidates) {
+      const host = hostOf(candidate.source_url);
+      if (!host) continue;
+      if (!domains.has(host)) domains.set(host, new Set());
+      domains.get(host).add(candidate.source_url);
+    }
+    members.push({
+      member_id: snapshot.member_id, member_role: snapshot.member_role, inventory_id: snapshot.id,
+      coverage: coverageOf(snapshot), ...memberSplit(domains, labelFor, labels, limit),
+    });
+  }
+
+  const reportHash = await discoveryHash([1, 'competitor_referring_domain_mix',
+    parsedLanes.map(lane => lane.lane), validated.map(s => [s.id, s.content_hash]), limit, skew]);
+  return {
+    metadata: {
+      v: 1, kind: 'referring_domain_mix', set_id: set?.id ?? null, set_revision: set?.revision ?? null,
+      comparison, ranking_scope: 'selected_inventory_datasets', whole_web_coverage: false,
+      absence_claim: 'not_supported',
+      // Labels are the customer's own imported vocabulary, applied to domains their rows named.
+      label_source: 'imported_row_labels', labels, lanes: parsedLanes.map(lane => lane.lane),
+      selected_inventory_ids: validated.map(s => s.id),
+      coverage_by_inventory: Object.fromEntries(validated.map(s => [s.id, coverageOf(s)])),
+      unselected_member_ids: set ? set.members.filter(m => !validated.some(s => s.member_id === m.id)).map(m => m.id).sort() : [],
+      competitor_side_available: validated.length > 0,
+      // With no set there is no competitor scope at all, and the report says which side is empty.
+      competitor_side_state: validated.length ? 'selected_inventories' : (set ? 'no_inventories_selected' : 'no_approved_competitor_set'),
+      report_hash: reportHash, limit,
+    },
+    ours,
+    members,
+  };
+}
+
+/**
+ * A change between two runs renders as a DIFF (the watch pattern), never a fresh page.
+ * Comparable means the same lanes and the same selected datasets: a diff across different
+ * datasets would present provider index churn as link acquisition, which is DP-0003's
+ * founding failure mode.
+ */
+export function diffDomainMixReports(previous, current) {
+  if (!previous || !current) return { comparable: false, reason: 'a report is missing' };
+  const sameLanes = JSON.stringify(previous.metadata?.lanes) === JSON.stringify(current.metadata?.lanes);
+  const sameDatasets = JSON.stringify(previous.metadata?.selected_inventory_ids) === JSON.stringify(current.metadata?.selected_inventory_ids);
+  if (!sameLanes || !sameDatasets) {
+    return { comparable: false, reason: sameLanes ? 'selected datasets differ; re-baseline rather than diff across datasets' : 'lanes differ' };
+  }
+  const sides = [], changes = [];
+  const side = (before, after, who) => {
+    const beforeHosts = before?.domain_classes ?? {};
+    const afterHosts = after?.domain_classes ?? {};
+    for (const [host, klass] of Object.entries(afterHosts)) if (!(host in beforeHosts)) changes.push({ who, direction: 'added', host, class: klass });
+    for (const [host, klass] of Object.entries(beforeHosts)) if (!(host in afterHosts)) changes.push({ who, direction: 'removed', host, class: klass });
+    sides.push({
+      who,
+      referring_domains: { from: before?.referring_domains ?? null, to: after?.referring_domains ?? null },
+      generic_share_of_labelled: { from: before?.generic_share_of_labelled ?? null, to: after?.generic_share_of_labelled ?? null },
+      label_coverage: { from: before?.label_coverage ?? null, to: after?.label_coverage ?? null },
+    });
+  };
+  (previous.ours ?? []).forEach((before, index) => side(before, current.ours?.[index], `ours:${before.lane}`));
+  (previous.members ?? []).forEach((before, index) => side(before, current.members?.[index], before.member_id));
+  return { comparable: true, v: 1, generated_from: current.metadata.report_hash, sides, changes };
+}
+
+/** The watch-pattern rendering: `from -> to` per side, one line per changed domain. */
+export function renderDomainMixDiff(diff, out = console.log) {
+  if (!diff.comparable) { out(`not comparable: ${diff.reason}`); return 0; }
+  for (const row of diff.sides) {
+    out(`${row.who}  referring-domains ${row.referring_domains.from} -> ${row.referring_domains.to}`
+      + `  generic-share ${row.generic_share_of_labelled.from} -> ${row.generic_share_of_labelled.to}`
+      + `  label-coverage ${row.label_coverage.from} -> ${row.label_coverage.to}`);
+  }
+  for (const change of diff.changes) out(`  ${change.direction === 'added' ? '+' : '-'} ${change.class} ${change.host}  (${change.who})`);
+  if (!diff.changes.length) out('  no referring-domain changes');
+  return 0;
+}