@nebulacomponents/citable 1.5.1 → 1.7.0

package/src/connectors/ga4.js

@@ -0,0 +1,59 @@
+import { providerRequest } from './http.js';
+
+const DATA = 'https://analyticsdata.googleapis.com/v1beta';
+const ADMIN = 'https://analyticsadmin.googleapis.com/v1beta';
+const METRICS = {
+  sessions: { unit: 'count', value_type: 'integer' },
+  engagedSessions: { unit: 'count', value_type: 'integer' },
+  keyEvents: { unit: 'count', value_type: 'number' },
+  totalRevenue: { unit: 'currency', value_type: 'number' },
+};
+const DIMENSIONS = new Set(['date', 'landingPagePlusQueryString', 'sessionDefaultChannelGroup', 'country', 'deviceCategory']);
+
+export const ga4Connector = {
+  provider: 'ga4', defaultCredentialEnv: 'GA4_ACCESS_TOKEN', readOnlyScopes: ['https://www.googleapis.com/auth/analytics.readonly'],
+  describeMetrics() { return METRICS; },
+  async discoverProperties(context) {
+    const properties = [];
+    let pageToken;
+    do {
+      const query = `?pageSize=200${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
+      const result = await providerRequest(`${ADMIN}/accountSummaries${query}`, context);
+      for (const account of result.accountSummaries || []) for (const property of account.propertySummaries || []) {
+        properties.push({ property_id: property.property.replace(/^properties\//, ''), display_name: property.displayName, account: account.account });
+      }
+      pageToken = result.nextPageToken;
+    } while (pageToken);
+    return properties;
+  },
+  async validateConnection(connection, context) {
+    const properties = await this.discoverProperties(context);
+    return { valid: properties.some((item) => item.property_id === String(connection.property_id)), properties };
+  },
+  async sync(connection, metrics, { startDate, endDate, ...context }) {
+    const dimensions = [...new Set(metrics.flatMap((metric) => metric.dimensions).filter((item) => DIMENSIONS.has(item)))];
+    if (!dimensions.includes('date')) dimensions.unshift('date');
+    if (!dimensions.includes('landingPagePlusQueryString')) dimensions.push('landingPagePlusQueryString');
+    const rows = [];
+    const limitations = [];
+    let offset = 0;
+    do {
+      const result = await providerRequest(`${DATA}/properties/${encodeURIComponent(connection.property_id)}:runReport`, {
+        ...context, method: 'POST', body: {
+        dateRanges: [{ startDate, endDate }], dimensions: dimensions.map((name) => ({ name })),
+        metrics: metrics.map((metric) => ({ name: metric.external_name })), limit: '100000', offset: String(offset),
+        dimensionFilter: { filter: { fieldName: 'sessionDefaultChannelGroup', stringFilter: { matchType: 'EXACT', value: 'Organic Search' } } },
+        },
+      });
+      for (const row of result.rows || []) {
+        const values = Object.fromEntries(dimensions.map((name, index) => [name === 'landingPagePlusQueryString' ? 'url' : name, row.dimensionValues[index]?.value]));
+        const date = values.date?.replace(/^(\d{4})(\d{2})(\d{2})$/, '$1-$2-$3');
+        rows.push(...metrics.map((metric, index) => ({ metric, value: Number(row.metricValues[index]?.value), dimensions: values, observed_at: `${date}T00:00:00.000Z` })));
+      }
+      if (result.metadata?.dataLossFromOtherRow) limitations.push('Provider reports data loss from the (other) row.');
+      offset += (result.rows || []).length;
+      if (!result.rowCount || offset >= result.rowCount || !(result.rows || []).length) break;
+    } while (true);
+    return { rows, cursor: endDate, limitations: [...new Set([...limitations, 'GA4 reporting identity, thresholding, attribution, and consent configuration affect results.', 'Sync is filtered to Organic Search sessions.'])] };
+  },
+};

package/src/connectors/gsc.js

@@ -0,0 +1,41 @@
+import { providerRequest } from './http.js';
+
+const BASE = 'https://www.googleapis.com/webmasters/v3';
+const METRICS = {
+  clicks: { unit: 'count', value_type: 'integer' },
+  impressions: { unit: 'count', value_type: 'integer' },
+  ctr: { unit: 'ratio', value_type: 'number' },
+  position: { unit: 'position', value_type: 'number' },
+};
+const DIMENSIONS = new Set(['date', 'query', 'page', 'country', 'device', 'searchAppearance']);
+
+export const gscConnector = {
+  provider: 'gsc', defaultCredentialEnv: 'GSC_ACCESS_TOKEN', readOnlyScopes: ['https://www.googleapis.com/auth/webmasters.readonly'],
+  describeMetrics() { return METRICS; },
+  async discoverProperties(context) {
+    const result = await providerRequest(`${BASE}/sites`, context);
+    return (result.siteEntry || []).map((item) => ({ property_id: item.siteUrl, permission: item.permissionLevel }));
+  },
+  async validateConnection(connection, context) {
+    const properties = await this.discoverProperties(context);
+    return { valid: properties.some((item) => item.property_id === connection.property_id), properties };
+  },
+  async sync(connection, metrics, { startDate, endDate, ...context }) {
+    const dimensions = [...new Set(metrics.flatMap((metric) => metric.dimensions).filter((item) => DIMENSIONS.has(item)))];
+    if (!dimensions.includes('date')) dimensions.unshift('date');
+    const rows = [];
+    let startRow = 0;
+    do {
+      const result = await providerRequest(`${BASE}/sites/${encodeURIComponent(connection.property_id)}/searchAnalytics/query`, {
+        ...context, method: 'POST', body: { startDate, endDate, dimensions, rowLimit: 25000, startRow, dataState: 'final' },
+      });
+      for (const row of result.rows || []) {
+        const values = Object.fromEntries(dimensions.map((name, index) => [name === 'page' ? 'url' : name, row.keys[index]]));
+        for (const metric of metrics) rows.push({ metric, value: row[metric.external_name], dimensions: values, observed_at: `${values.date}T00:00:00.000Z` });
+      }
+      if ((result.rows || []).length < 25000) break;
+      startRow += 25000;
+    } while (true);
+    return { rows, cursor: endDate, limitations: ['Search Console privacy filtering and aggregation apply.', 'Search Analytics does not guarantee every data row; the API can return top rows only.', 'Final data can still be revised by the provider.'] };
+  },
+};

package/src/connectors/http.js

@@ -0,0 +1,19 @@
+const RETRYABLE = new Set([429, 500, 502, 503, 504]);
+
+export async function providerRequest(url, { token, fetchImpl = globalThis.fetch, method = 'GET', body, attempts = 3 } = {}) {
+  if (!token) throw Object.assign(new Error('authorization is not configured'), { connectorState: 'not_configured' });
+  let response;
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    response = await fetchImpl(url, {
+      method,
+      headers: { authorization: `Bearer ${token}`, accept: 'application/json', ...(body ? { 'content-type': 'application/json' } : {}) },
+      body: body ? JSON.stringify(body) : undefined,
+    });
+    if (response.ok) return response.json();
+    const detail = (await response.text()).slice(0, 300);
+    if (!RETRYABLE.has(response.status) || attempt === attempts) {
+      const states = { 401: 'authorization_expired', 403: 'permission_denied', 429: 'quota_limited' };
+      throw Object.assign(new Error(`provider returned ${response.status}: ${detail}`), { connectorState: states[response.status] || 'temporarily_unavailable' });
+    }
+  }
+}

package/src/connectors/index.js

@@ -0,0 +1,14 @@
+import { gscConnector } from './gsc.js';
+import { ga4Connector } from './ga4.js';
+
+const CONNECTORS = new Map([gscConnector, ga4Connector].map((connector) => [connector.provider, connector]));
+
+export function getConnector(provider) {
+  const connector = CONNECTORS.get(String(provider).toLowerCase());
+  if (!connector) throw new Error(`unsupported connector provider: ${provider}`);
+  return connector;
+}
+
+export function listConnectors() {
+  return [...CONNECTORS.values()].map((item) => ({ provider: item.provider, credential_env: item.defaultCredentialEnv, scopes: item.readOnlyScopes, metrics: Object.keys(item.describeMetrics()) }));
+}

package/src/observations/authority.js

@@ -0,0 +1,13 @@
+const DEFAULTS = {
+  live_api: ['provider_published', 'direct_api', 'transport_authenticated', 'unknown'],
+  browser: ['synthetic', 'browser_capture', 'checksum_protected_only', 'single_observation'],
+  owner_import: ['owner_controlled', 'owner_export', 'checksum_protected_only', 'unknown'],
+  synthetic_fetch: ['synthetic', 'synthetic_probe', 'checksum_protected_only', 'single_observation'],
+  static_analysis: ['inferred', 'static_analysis', 'checksum_protected_only', 'single_observation'],
+  human_review: ['owner_controlled', 'manual_entry', 'owner_attested', 'single_observation'],
+};
+
+export function evidenceAuthority(method, overrides = {}) {
+  const [source_authority, collection_authority, authenticity_status, representativeness] = DEFAULTS[method] || DEFAULTS.static_analysis;
+  return { source_authority, collection_authority, authenticity_status, representativeness, ...overrides };
+}

package/src/observations/common.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { createRun } from '../evidence/run.js';
 import { nowIso, readJson, sha256 } from '../shared/io.js';
 import { validateAgainst } from '../shared/schemaValidator.js';
+import { evidenceAuthority } from './authority.js';
 
 export function readInput(input) {
   if (!input) throw new Error('--input <json> is required for this collector');
@@ -10,12 +11,13 @@ export function readInput(input) {
   return { raw: fs.readFileSync(input, 'utf8'), value: readJson(input), file: path.resolve(input) };
 }
 
-export function envelope(kind, data, { method, source, state = 'observed', confidence = 'confirmed', limitations = [], raw } = {}) {
+export function envelope(kind, data, { method, source, state = 'observed', confidence = 'confirmed', limitations = [], authority, raw } = {}) {
   const evidence = raw ?? JSON.stringify(data);
   const item = {
     observation_id: `OBS-${kind.toUpperCase()}-${sha256(evidence).slice(0, 16)}`,
     kind, state, collected_at: nowIso(), collection_method: method,
     confidence, source, evidence_hash: sha256(evidence), data, limitations,
+    authority: evidenceAuthority(method, authority),
   };
   const check = validateAgainst('observation.schema.json', item);
   if (!check.valid) throw new Error(`observation violates contract: ${check.errors.join('; ')}`);

package/src/observations/crawlerIdentity.js

@@ -0,0 +1,60 @@
+import net from 'node:net';
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+function cidrMembership(sourceIp, cidrs) {
+  if (!sourceIp) return 'not_tested';
+  const family = net.isIP(sourceIp);
+  if (!family) return 'invalid';
+  const block = new net.BlockList();
+  for (const cidr of cidrs || []) {
+    const [network, prefixText] = String(cidr).split('/');
+    const prefix = Number(prefixText), rangeFamily = net.isIP(network);
+    if (rangeFamily === family && Number.isInteger(prefix)) block.addSubnet(network, prefix, family === 4 ? 'ipv4' : 'ipv6');
+  }
+  return block.check(sourceIp, family === 4 ? 'ipv4' : 'ipv6') ? 'matched' : 'not_matched';
+}
+
+export function crawlerIdentity(row, { ranges = {}, rangeSources = {}, collector = 'owner_import' } = {}) {
+  const ua = row.user_agent, cidrs = ranges[ua] || [], source = rangeSources[ua] || {};
+  const sourceSnapshot = source.source_snapshot;
+  const expectedChecksum = sourceSnapshot == null ? null : sha256(typeof sourceSnapshot === 'string' ? sourceSnapshot : JSON.stringify(sourceSnapshot));
+  const rangeSource = {
+    url: source.url || null,
+    retrieved_at: source.retrieved_at || null,
+    authority: source.authority || 'unknown',
+    checksum: source.checksum || null,
+    checksum_scope: sourceSnapshot == null ? 'not_captured' : 'source_snapshot',
+    checksum_valid: Boolean(source.checksum && expectedChecksum && source.checksum === expectedChecksum),
+  };
+  const membership = cidrMembership(row.source_ip, cidrs);
+  const dns = { status: row.dns_status || 'not_tested', method: row.dns_method || null };
+  const edge = { observed: row.edge_observed === true, decision: row.edge_decision || null };
+  const origin = { observed: row.origin_observed === true, status: row.origin_status ?? null };
+  const stages = ['declared'];
+  if (row.synthetic === true) stages.push('synthetically_observed');
+  if (membership === 'matched') stages.push('range_matched');
+  if (dns.status === 'verified') stages.push('dns_verified');
+  if (edge.observed) stages.push('edge_observed');
+  if (origin.observed) stages.push('origin_observed');
+  const contradictory = membership === 'not_matched' || dns.status === 'failed' || (row.claimed_verified === true && (!rangeSource.checksum_valid || membership !== 'matched'));
+  const complete = identityProvider(row, source) !== 'unknown' && row.synthetic !== true && rangeSource.url && rangeSource.retrieved_at && rangeSource.authority === 'provider_published' && rangeSource.checksum_scope === 'source_snapshot' && rangeSource.checksum_valid && membership === 'matched' && dns.status === 'verified' && edge.observed && origin.observed;
+  if (contradictory) stages.push('contradictory');
+  else if (complete) stages.push('fully_verified');
+  else stages.push('insufficient_evidence');
+  const precedence = ['contradictory', 'fully_verified', 'origin_observed', 'edge_observed', 'dns_verified', 'range_matched', 'synthetically_observed', 'declared'];
+  const verificationStatus = precedence.find((state) => stages.includes(state));
+  const identity = {
+    provider: identityProvider(row, source), declared_user_agent: ua,
+    observed_source_ip: row.source_ip || null, range_source: rangeSource,
+    cidr_membership: membership, dns, edge, origin, region: row.region || null,
+    collector, stages, verification_status: verificationStatus,
+  };
+  const check = validateAgainst('crawler-identity.schema.json', identity);
+  if (!check.valid) throw new Error(`crawler identity violates contract: ${check.errors.join('; ')}`);
+  return identity;
+}
+
+function identityProvider(row, source) {
+  return row.provider || source.provider || 'unknown';
+}