@nebulacomponents/citable 1.5.1 → 1.6.0

package/skill/commands/measurement.md

@@ -10,7 +10,7 @@ imports, and a disclosed custom citation-adapter protocol. It does not claim
 Bing AI Performance automation or equivalence between an API adapter and a
 consumer answer product. Preserve collector and product mode on every result.
 
-# implemented metric and objective workflow
+# implemented metric, connector, and objective workflow
 
 `citable metrics import --provider <name> --input <csv|json>` validates every
 row against a declared entry in `metrics.yaml` and writes immutable metric
@@ -27,6 +27,21 @@ absence of provider credentials is `not_configured`, not a finding and not
 evidence of zero activity. Credentials must never enter registries or evidence
 packages.
 
+Live GSC and GA4 connections are explicitly optional. Configure only a
+property identifier and the name of the environment variable that holds its
+OAuth token; never place the token itself in `.citable/connections.yaml`.
+`citable connect configure` and `disconnect` are dry runs unless `--write` is
+used. `connect discover`, `validate`, and `sync` require read-only provider
+authorization. `sync` writes declared metric observations with provider
+limitations and advances a non-secret cursor. Provider errors are recorded as
+connection state, not converted into zero observations or audit findings.
+
+The GSC adapter collects final Search Analytics rows and inherits Search
+Console aggregation, privacy filtering, and top-row availability limits. The GA4 adapter collects Data API
+rows restricted to the `Organic Search` default channel group and inherits the
+property's reporting identity, thresholding, attribution, and consent choices.
+Neither source establishes that an intervention caused an observed change.
+
 # measure seo
 Inputs the operator exports (Search Console/Bing/analytics CSV or JSON).
 Workflow: validate segmentation (brand vs non-brand, intent, page type,

package/src/cli/index.js

@@ -16,6 +16,7 @@ import { observe } from '../commands/observe.js';
 import { applyRemediation } from '../commands/applyRemediation.js';
 import { monitor } from '../commands/monitor.js';
 import { evaluateObjective, importMetrics, initializeObjective, validateObjectives } from '../commands/measurement.js';
+import { configureConnection, connectionStatus, discoverConnections, disconnectConnection, syncConnection, validateConnection } from '../commands/connect.js';
 
 const HELP = `citable — SEO / AEO / GEO audit, remediation, validation, and governance
 
@@ -43,6 +44,12 @@ Commands
   apply                     Apply a reviewed, hash-locked remediation spec
   monitor [runA runB]       Compare observation runs and emit regression alerts
   metrics import            Import declared metric observations from CSV/JSON
+  connect status            List optional connectors and configured connections
+  connect configure         Configure non-secret connection state (--write to save)
+  connect discover          Discover provider properties using environment auth
+  connect validate          Verify configured property access
+  connect sync              Collect declared metrics into immutable observations
+  connect disconnect        Remove optional connection state (--write to confirm)
   objectives init           Validate/add one objective from --input (--write to save)
   objectives validate       Validate objective contracts and metric references
   evaluate [objective-id]   Compare objective baseline and evaluation windows
@@ -54,6 +61,11 @@ Options
   --ref-date <YYYY-MM-DD>   Reference date for expiry/staleness checks (default: today)
   --input <file>            Import file or remediation specification
   --provider <name>         Provider label for imported observations
+  --connection-id <id>      Connection registry identifier
+  --property-id <id>        Provider property or site identifier
+  --credential-env <name>   Environment variable containing the access token
+  --start-date <YYYY-MM-DD> Connector collection window start
+  --end-date <YYYY-MM-DD>   Connector collection window end
   --api-key <key>           API key (prefer provider environment variables)
   --site-url <property>     Search Console property for live URL inspection
   --access-token <token>    OAuth token (prefer provider environment variables)
@@ -76,6 +88,11 @@ function parseArgs(argv) {
     else if (a === '--ref-date') args.refDate = argv[++i];
     else if (a === '--input') args.input = argv[++i];
     else if (a === '--provider') args.provider = argv[++i];
+    else if (a === '--connection-id') args.connectionId = argv[++i];
+    else if (a === '--property-id') args.propertyId = argv[++i];
+    else if (a === '--credential-env') args.credentialEnv = argv[++i];
+    else if (a === '--start-date') args.startDate = argv[++i];
+    else if (a === '--end-date') args.endDate = argv[++i];
     else if (a === '--api-key') args.apiKey = argv[++i];
     else if (a === '--site-url') args.siteUrl = argv[++i];
     else if (a === '--access-token') args.accessToken = argv[++i];
@@ -178,6 +195,30 @@ export async function main(argv = process.argv.slice(2), options = {}) {
         out(args, `metrics import: ${r.summary.total} observation(s) from ${args.provider}\nEvidence package: ${r.dir}`, r);
         break;
       }
+      case 'connect': {
+        const mode = args._[0];
+        if (mode === 'status') {
+          const r = connectionStatus(root);
+          out(args, `connect status: ${r.connections.length} configured connection(s)\n` + r.available.map((item) => `  ${item.provider}: token via ${item.credential_env}`).join('\n'), r);
+        } else if (mode === 'configure') {
+          const r = configureConnection(root, args);
+          out(args, `connect configure: ${r.connection.connection_id} ${r.written ? 'written' : 'valid (dry run; use --write to save)'}`, r);
+        } else if (mode === 'discover') {
+          const r = await discoverConnections(root, args);
+          out(args, `connect discover ${r.provider}: ${r.properties.length} accessible property/properties`, r);
+        } else if (mode === 'validate') {
+          const r = await validateConnection(root, args);
+          out(args, `connect validate ${r.connection_id}: ${r.valid ? 'accessible' : 'not accessible'}`, r);
+          if (!r.valid) process.exitCode = 1;
+        } else if (mode === 'sync') {
+          const r = await syncConnection(root, args);
+          out(args, `connect sync ${r.connection_id}: ${r.summary.total} metric observation(s)\nEvidence package: ${r.dir}`, r);
+        } else if (mode === 'disconnect') {
+          const r = disconnectConnection(root, args);
+          out(args, `connect disconnect ${r.connection_id}: ${r.disconnected ? 'removed' : 'dry run; use --write to remove'}`, r);
+        } else throw new Error('usage: citable connect <status|configure|discover|validate|sync> [options]');
+        break;
+      }
       case 'objectives': {
         const mode = args._[0];
         if (mode === 'init') {

package/src/commands/connect.js

@@ -0,0 +1,90 @@
+import { loadRegistries, saveRegistry } from '../registries/index.js';
+import { envelope, observationRun } from '../observations/common.js';
+import { getConnector, listConnectors } from '../connectors/index.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+function findConnection(root, connectionId) {
+  const loaded = loadRegistries(root);
+  if (loaded.problems.length) throw new Error(`registry validation failed: ${loaded.problems.join('; ')}`);
+  const connection = loaded.registries.connections.entries.find((item) => item.connection_id === connectionId);
+  if (!connection) throw new Error(`connection not found: ${connectionId}`);
+  return { ...loaded, connection };
+}
+
+function contextFor(connection, connector, options) {
+  const credentialEnv = connection.credential_env || connector.defaultCredentialEnv;
+  return { token: options.accessToken || options.env?.[credentialEnv] || process.env[credentialEnv], fetchImpl: options.fetchImpl, credentialEnv };
+}
+
+export function connectionStatus(root) {
+  const { registries, problems } = loadRegistries(root);
+  if (problems.length) throw new Error(`registry validation failed: ${problems.join('; ')}`);
+  return { available: listConnectors(), connections: registries.connections.entries };
+}
+
+export function configureConnection(root, { provider, connectionId, propertyId, credentialEnv, write = false }) {
+  if (!provider || !connectionId || !propertyId) throw new Error('connect configure requires --provider, --connection-id, and --property-id');
+  const connector = getConnector(provider);
+  const { registries, problems } = loadRegistries(root);
+  if (problems.length) throw new Error(`registry validation failed: ${problems.join('; ')}`);
+  const entry = { connection_id: connectionId, provider: connector.provider, property_id: String(propertyId), state: 'configured', authentication: 'environment', credential_env: credentialEnv || connector.defaultCredentialEnv, scopes: connector.readOnlyScopes, last_synchronized_at: null, cursor: null, limitations: [] };
+  const entries = registries.connections.entries.filter((item) => item.connection_id !== connectionId);
+  const candidate = { ...registries.connections, entries: [...entries, entry] };
+  const check = validateAgainst('connection.schema.json', candidate);
+  if (!check.valid) throw new Error(`connection violates contract: ${check.errors.join('; ')}`);
+  if (write) saveRegistry(root, 'connections', candidate);
+  return { connection: entry, written: write };
+}
+
+export async function discoverConnections(root, { provider, accessToken, env, fetchImpl } = {}) {
+  const connector = getConnector(provider);
+  const context = { token: accessToken || env?.[connector.defaultCredentialEnv] || process.env[connector.defaultCredentialEnv], fetchImpl };
+  return { provider: connector.provider, properties: await connector.discoverProperties(context), credential_env: connector.defaultCredentialEnv };
+}
+
+export async function validateConnection(root, { connectionId, accessToken, env, fetchImpl } = {}) {
+  const { connection } = findConnection(root, connectionId);
+  const connector = getConnector(connection.provider);
+  const result = await connector.validateConnection(connection, contextFor(connection, connector, { accessToken, env, fetchImpl }));
+  return { connection_id: connectionId, provider: connection.provider, state: result.valid ? 'configured' : 'permission_denied', ...result };
+}
+
+export function disconnectConnection(root, { connectionId, write = false } = {}) {
+  const loaded = findConnection(root, connectionId);
+  const candidate = { ...loaded.registries.connections, entries: loaded.registries.connections.entries.filter((item) => item.connection_id !== connectionId) };
+  if (write) saveRegistry(root, 'connections', candidate);
+  return { connection_id: connectionId, disconnected: write, state: write ? 'not_configured' : loaded.connection.state };
+}
+
+export async function syncConnection(root, { connectionId, startDate, endDate, accessToken, env, fetchImpl } = {}) {
+  if (!startDate || !endDate) throw new Error('connect sync requires --start-date and --end-date');
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(startDate) || !/^\d{4}-\d{2}-\d{2}$/.test(endDate) || Number.isNaN(Date.parse(`${startDate}T00:00:00Z`)) || Number.isNaN(Date.parse(`${endDate}T00:00:00Z`))) throw new Error('connector dates must use valid YYYY-MM-DD values');
+  if (startDate > endDate) throw new Error('connector start-date must not be after end-date');
+  const loaded = findConnection(root, connectionId);
+  const connector = getConnector(loaded.connection.provider);
+  const metrics = loaded.registries.metrics.entries.filter((item) => item.provider.toLowerCase() === connector.provider && item.status !== 'deprecated');
+  if (!metrics.length) throw new Error(`no active metrics declared for provider ${connector.provider}`);
+  const supported = connector.describeMetrics();
+  const unsupported = metrics.filter((item) => !supported[item.external_name]);
+  if (unsupported.length) throw new Error(`unsupported ${connector.provider} metrics: ${unsupported.map((item) => item.external_name).join(', ')}`);
+  let result;
+  try {
+    result = await connector.sync(loaded.connection, metrics, { startDate, endDate, ...contextFor(loaded.connection, connector, { accessToken, env, fetchImpl }) });
+  } catch (error) {
+    if (error.connectorState) {
+      const failed = { ...loaded.registries.connections, entries: loaded.registries.connections.entries.map((item) => item.connection_id === connectionId ? { ...item, state: error.connectorState, limitations: [`${connector.provider} synchronization did not complete; inspect command output and provider access.`] } : item) };
+      saveRegistry(root, 'connections', failed);
+    }
+    throw error;
+  }
+  const observations = result.rows.map((row) => envelope('metric', {
+    metric_id: row.metric.metric_id, provider: connector.provider, external_name: row.metric.external_name,
+    value: Number(row.value), unit: row.metric.unit === 'custom' ? row.metric.custom_unit : row.metric.unit,
+    aggregation: row.metric.aggregation, observed_at: row.observed_at, period_start: row.observed_at,
+    period_end: row.observed_at, dimensions: row.dimensions,
+  }, { method: 'live_api', source: connector.provider, limitations: [...row.metric.limitations, ...result.limitations] }));
+  const run = observationRun(root, 'connect sync', loaded.connection.property_id, observations);
+  const updated = { ...loaded.registries.connections, entries: loaded.registries.connections.entries.map((item) => item.connection_id === connectionId ? { ...item, state: 'synchronized', last_synchronized_at: new Date().toISOString(), cursor: result.cursor, limitations: result.limitations } : item) };
+  saveRegistry(root, 'connections', updated);
+  return { ...run, connection_id: connectionId, provider: connector.provider };
+}

package/src/connectors/ga4.js

@@ -0,0 +1,59 @@
+import { providerRequest } from './http.js';
+
+const DATA = 'https://analyticsdata.googleapis.com/v1beta';
+const ADMIN = 'https://analyticsadmin.googleapis.com/v1beta';
+const METRICS = {
+  sessions: { unit: 'count', value_type: 'integer' },
+  engagedSessions: { unit: 'count', value_type: 'integer' },
+  keyEvents: { unit: 'count', value_type: 'number' },
+  totalRevenue: { unit: 'currency', value_type: 'number' },
+};
+const DIMENSIONS = new Set(['date', 'landingPagePlusQueryString', 'sessionDefaultChannelGroup', 'country', 'deviceCategory']);
+
+export const ga4Connector = {
+  provider: 'ga4', defaultCredentialEnv: 'GA4_ACCESS_TOKEN', readOnlyScopes: ['https://www.googleapis.com/auth/analytics.readonly'],
+  describeMetrics() { return METRICS; },
+  async discoverProperties(context) {
+    const properties = [];
+    let pageToken;
+    do {
+      const query = `?pageSize=200${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
+      const result = await providerRequest(`${ADMIN}/accountSummaries${query}`, context);
+      for (const account of result.accountSummaries || []) for (const property of account.propertySummaries || []) {
+        properties.push({ property_id: property.property.replace(/^properties\//, ''), display_name: property.displayName, account: account.account });
+      }
+      pageToken = result.nextPageToken;
+    } while (pageToken);
+    return properties;
+  },
+  async validateConnection(connection, context) {
+    const properties = await this.discoverProperties(context);
+    return { valid: properties.some((item) => item.property_id === String(connection.property_id)), properties };
+  },
+  async sync(connection, metrics, { startDate, endDate, ...context }) {
+    const dimensions = [...new Set(metrics.flatMap((metric) => metric.dimensions).filter((item) => DIMENSIONS.has(item)))];
+    if (!dimensions.includes('date')) dimensions.unshift('date');
+    if (!dimensions.includes('landingPagePlusQueryString')) dimensions.push('landingPagePlusQueryString');
+    const rows = [];
+    const limitations = [];
+    let offset = 0;
+    do {
+      const result = await providerRequest(`${DATA}/properties/${encodeURIComponent(connection.property_id)}:runReport`, {
+        ...context, method: 'POST', body: {
+        dateRanges: [{ startDate, endDate }], dimensions: dimensions.map((name) => ({ name })),
+        metrics: metrics.map((metric) => ({ name: metric.external_name })), limit: '100000', offset: String(offset),
+        dimensionFilter: { filter: { fieldName: 'sessionDefaultChannelGroup', stringFilter: { matchType: 'EXACT', value: 'Organic Search' } } },
+        },
+      });
+      for (const row of result.rows || []) {
+        const values = Object.fromEntries(dimensions.map((name, index) => [name === 'landingPagePlusQueryString' ? 'url' : name, row.dimensionValues[index]?.value]));
+        const date = values.date?.replace(/^(\d{4})(\d{2})(\d{2})$/, '$1-$2-$3');
+        rows.push(...metrics.map((metric, index) => ({ metric, value: Number(row.metricValues[index]?.value), dimensions: values, observed_at: `${date}T00:00:00.000Z` })));
+      }
+      if (result.metadata?.dataLossFromOtherRow) limitations.push('Provider reports data loss from the (other) row.');
+      offset += (result.rows || []).length;
+      if (!result.rowCount || offset >= result.rowCount || !(result.rows || []).length) break;
+    } while (true);
+    return { rows, cursor: endDate, limitations: [...new Set([...limitations, 'GA4 reporting identity, thresholding, attribution, and consent configuration affect results.', 'Sync is filtered to Organic Search sessions.'])] };
+  },
+};

package/src/connectors/gsc.js

@@ -0,0 +1,41 @@
+import { providerRequest } from './http.js';
+
+const BASE = 'https://www.googleapis.com/webmasters/v3';
+const METRICS = {
+  clicks: { unit: 'count', value_type: 'integer' },
+  impressions: { unit: 'count', value_type: 'integer' },
+  ctr: { unit: 'ratio', value_type: 'number' },
+  position: { unit: 'position', value_type: 'number' },
+};
+const DIMENSIONS = new Set(['date', 'query', 'page', 'country', 'device', 'searchAppearance']);
+
+export const gscConnector = {
+  provider: 'gsc', defaultCredentialEnv: 'GSC_ACCESS_TOKEN', readOnlyScopes: ['https://www.googleapis.com/auth/webmasters.readonly'],
+  describeMetrics() { return METRICS; },
+  async discoverProperties(context) {
+    const result = await providerRequest(`${BASE}/sites`, context);
+    return (result.siteEntry || []).map((item) => ({ property_id: item.siteUrl, permission: item.permissionLevel }));
+  },
+  async validateConnection(connection, context) {
+    const properties = await this.discoverProperties(context);
+    return { valid: properties.some((item) => item.property_id === connection.property_id), properties };
+  },
+  async sync(connection, metrics, { startDate, endDate, ...context }) {
+    const dimensions = [...new Set(metrics.flatMap((metric) => metric.dimensions).filter((item) => DIMENSIONS.has(item)))];
+    if (!dimensions.includes('date')) dimensions.unshift('date');
+    const rows = [];
+    let startRow = 0;
+    do {
+      const result = await providerRequest(`${BASE}/sites/${encodeURIComponent(connection.property_id)}/searchAnalytics/query`, {
+        ...context, method: 'POST', body: { startDate, endDate, dimensions, rowLimit: 25000, startRow, dataState: 'final' },
+      });
+      for (const row of result.rows || []) {
+        const values = Object.fromEntries(dimensions.map((name, index) => [name === 'page' ? 'url' : name, row.keys[index]]));
+        for (const metric of metrics) rows.push({ metric, value: row[metric.external_name], dimensions: values, observed_at: `${values.date}T00:00:00.000Z` });
+      }
+      if ((result.rows || []).length < 25000) break;
+      startRow += 25000;
+    } while (true);
+    return { rows, cursor: endDate, limitations: ['Search Console privacy filtering and aggregation apply.', 'Search Analytics does not guarantee every data row; the API can return top rows only.', 'Final data can still be revised by the provider.'] };
+  },
+};

package/src/connectors/http.js

@@ -0,0 +1,19 @@
+const RETRYABLE = new Set([429, 500, 502, 503, 504]);
+
+export async function providerRequest(url, { token, fetchImpl = globalThis.fetch, method = 'GET', body, attempts = 3 } = {}) {
+  if (!token) throw Object.assign(new Error('authorization is not configured'), { connectorState: 'not_configured' });
+  let response;
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    response = await fetchImpl(url, {
+      method,
+      headers: { authorization: `Bearer ${token}`, accept: 'application/json', ...(body ? { 'content-type': 'application/json' } : {}) },
+      body: body ? JSON.stringify(body) : undefined,
+    });
+    if (response.ok) return response.json();
+    const detail = (await response.text()).slice(0, 300);
+    if (!RETRYABLE.has(response.status) || attempt === attempts) {
+      const states = { 401: 'authorization_expired', 403: 'permission_denied', 429: 'quota_limited' };
+      throw Object.assign(new Error(`provider returned ${response.status}: ${detail}`), { connectorState: states[response.status] || 'temporarily_unavailable' });
+    }
+  }
+}

package/src/connectors/index.js

@@ -0,0 +1,14 @@
+import { gscConnector } from './gsc.js';
+import { ga4Connector } from './ga4.js';
+
+const CONNECTORS = new Map([gscConnector, ga4Connector].map((connector) => [connector.provider, connector]));
+
+export function getConnector(provider) {
+  const connector = CONNECTORS.get(String(provider).toLowerCase());
+  if (!connector) throw new Error(`unsupported connector provider: ${provider}`);
+  return connector;
+}
+
+export function listConnectors() {
+  return [...CONNECTORS.values()].map((item) => ({ provider: item.provider, credential_env: item.defaultCredentialEnv, scopes: item.readOnlyScopes, metrics: Object.keys(item.describeMetrics()) }));
+}