optbrew 0.0.0-stage → 0.1.2

package/skills/optbrew/scripts/core/metrics.mjs

@@ -0,0 +1,97 @@
+import { ContractError, validateEntity } from './contracts.mjs';
+
+function ratio(id, numerator, denominator, definition, panel, selection, evaluable = true) {
+  const measured = evaluable && denominator > 0;
+  const observation = { schema_version: '1.0.0', metric_id: id,
+    value_state: measured ? 'measured' : 'unknown', value: measured ? numerator / denominator : null,
+    numerator: measured ? numerator : null, denominator: measured ? denominator : null,
+    definition, filters: { engine: panel.engine, surface: panel.surface, model: panel.model,
+      search_enabled: panel.search_enabled, locale: panel.locale, region: panel.region,
+      ...(panel.collected_at ? { collected_at: panel.collected_at } : {}),
+      ...(panel.last_complete_date ? { last_complete_date: panel.last_complete_date } : {}) },
+    window: panel.window, panel_hash: panel.panel_hash, provider_definition: 'optbrew fixed AI response panel v0.1.0',
+    sample_selection: selection, source_ref: panel.source_ref, evidence_ref: panel.evidence_ref };
+  return validateEntity('MetricObservation', observation);
+}
+
+/** Six distinct panel metrics; response-level and citation/claim denominators never mix. */
+export function measurePanel(panel, responses, evaluations) {
+  if (!panel || !Array.isArray(panel.questions) || !panel.questions.length ||
+      new Set(panel.questions.map(q => q.id)).size !== panel.questions.length ||
+      !Number.isInteger(panel.repeats) || panel.repeats < 1 || !panel.panel_hash ||
+      typeof panel.search_enabled !== 'boolean' || !panel.target_entity ||
+      !Array.isArray(panel.target_domains) || !panel.target_domains.length ||
+      !Array.isArray(responses) || !Array.isArray(evaluations))
+    throw new ContractError('invalid fixed panel');
+  const planned = panel.questions.length * panel.repeats;
+  const slots = new Set(), responseIds = new Set(), valid = [];
+  for (const response of responses) {
+    const slot = `${response.question_id}\0${response.repeat}`;
+    if (!panel.questions.some(q => q.id === response.question_id) ||
+        !Number.isInteger(response.repeat) || response.repeat < 0 || response.repeat >= panel.repeats ||
+        slots.has(slot) || !response.response_id || responseIds.has(response.response_id) || typeof response.valid !== 'boolean')
+      throw new ContractError('response outside or duplicated in fixed panel');
+    slots.add(slot); responseIds.add(response.response_id);
+    if (response.valid) {
+      if (!Array.isArray(response.cited_urls) || typeof response.target_mentioned !== 'boolean' ||
+          !Array.isArray(response.brand_mentions)) throw new ContractError('valid response lacks evaluated fields');
+      for (const citedUrl of response.cited_urls) {
+        try { if (!['http:', 'https:'].includes(new URL(citedUrl).protocol)) throw new Error(); }
+        catch { throw new ContractError('valid response has invalid citation URL'); }
+      }
+      if (new Set(response.cited_urls).size !== response.cited_urls.length)
+        throw new ContractError('duplicate citation URL in response');
+      valid.push(response);
+    }
+  }
+  const selection = { kind: 'all', planned_responses: planned, valid_responses: valid.length,
+    unique_questions: panel.questions.length, inference: panel.questions.length < 10 ? 'exploratory' : 'descriptive' };
+  const targetDomains = panel.target_domains.map(domain => domain.toLowerCase());
+  const cited = response => (response.cited_urls ?? []).some(url => {
+    try { const host = new URL(url).hostname.toLowerCase(); return targetDomains.some(domain => host === domain || host.endsWith(`.${domain}`)); }
+    catch { return false; }
+  });
+  const citationCount = valid.filter(cited).length;
+  const mentionCount = valid.filter(r => r.target_mentioned === true).length;
+  const brands = new Set([panel.target_entity, ...(panel.competitor_entities ?? [])]);
+  const brandCount = valid.reduce((sum, r) => sum + [...new Set(r.brand_mentions)].filter(x => brands.has(x)).length, 0);
+  const targetBrandCount = valid.filter(r => r.brand_mentions.includes(panel.target_entity)).length;
+  const validIds = new Set(valid.map(r => r.response_id));
+  if (evaluations.some(e => !validIds.has(e.response_id)) ||
+      new Set(evaluations.map(e => e.response_id)).size !== evaluations.length)
+    throw new ContractError('duplicate evaluation or evaluation without valid panel response');
+  const responseById = new Map(valid.map(r => [r.response_id, r]));
+  const support = [];
+  const observedCitations = valid.reduce((sum, r) => sum + r.cited_urls.length, 0);
+  for (const evaluation of evaluations) {
+    if (evaluation.citations !== undefined && !Array.isArray(evaluation.citations))
+      throw new ContractError('citation evaluations must be an array');
+    const actual = new Set(responseById.get(evaluation.response_id).cited_urls);
+    const bound = new Set();
+    for (const citation of evaluation.citations ?? []) {
+      if (!citation || typeof citation.citation_url !== 'string' || !actual.has(citation.citation_url) ||
+          bound.has(citation.citation_url))
+        throw new ContractError('citation evaluation must bind one actual response citation');
+      bound.add(citation.citation_url);
+      if (citation.evaluable === true && citation.source_accessible === true && typeof citation.supported === 'boolean')
+        support.push(citation);
+    }
+  }
+  const facts = panel.fact_list_ref ? evaluations.flatMap(e => e.claims ?? []).filter(x =>
+    x.evaluable === true && typeof x.matches === 'boolean') : [];
+  return [
+    ratio('target_citation_response_rate', citationCount, valid.length, 'valid responses citing a target URL or domain / valid responses', panel, selection),
+    ratio('target_entity_mention_rate', mentionCount, valid.length, 'valid responses with evaluated target entity mention / valid responses', panel, selection),
+    ratio('citation_support_rate', support.filter(x => x.supported === true).length, support.length,
+      'supported evaluable linked citations / evaluable linked citations', panel,
+      { ...selection, observed_citations: observedCitations, evaluation_scope: support.length,
+        unknown_citations: observedCitations - support.length,
+        ...(support.length ? {} : { missing_reason: 'no_accessible_evaluable_citations' }) }, support.length > 0),
+    ratio('fact_consistency_rate', facts.filter(x => x.matches === true).length, facts.length,
+      'matching evaluable important claims / evaluable important claims', panel,
+      { ...selection, evaluation_scope: facts.length, ...(panel.fact_list_ref ? {} : { missing_reason: 'approved_fact_list_missing' }) }, facts.length > 0),
+    ratio('observed_brand_share', targetBrandCount, brandCount,
+      'target mentions once per response / mentions for fixed competitor set', panel, selection),
+    ratio('observation_completion_rate', valid.length, planned, 'valid responses / planned responses', panel, selection),
+  ];
+}

package/skills/optbrew/scripts/core/protocol.mjs

@@ -0,0 +1,125 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { ContractError, validateEntity, validateApplicabilityDecision } from './contracts.mjs';
+import { assertUrlInScope } from './scope.mjs';
+
+const registry = JSON.parse(readFileSync(new URL('../../references/check-registry.json', import.meta.url)));
+const knownFamilies = new Map(registry.families.map(x => [x.family_id, x]));
+
+function unique(values, label) {
+  if (values.some(x => typeof x !== 'string' || !x) || new Set(values).size !== values.length)
+    throw new ContractError(`${label}: blank or duplicate identifier`);
+}
+
+const compare = (a, b) => a < b ? -1 : a > b ? 1 : 0;
+const sameSet = (a, b) => a.length === b.length && a.every(value => b.includes(value));
+
+function stable(value) {
+  if (Array.isArray(value)) return value.map(stable);
+  if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(key => [key, stable(value[key])]));
+  return value;
+}
+
+/** Fix comparison inputs; run_id, timestamps and evidence paths are excluded. */
+export function freezeProtocol(context) {
+  validateEntity('RunContext', context);
+  const p = context.protocol;
+  for (const key of ['target', 'environment', 'evaluation_version', 'protocol_version'])
+    if (Object.hasOwn(p, key) && JSON.stringify(p[key]) !== JSON.stringify(context[key]))
+      throw new ContractError(`protocol ${key} differs from RunContext`);
+  unique(p.page_types.map(x => x.id), 'page_types');
+  unique(p.engines, 'engines');
+  unique(p.families.map(x => x.family_id), 'families');
+  unique(p.question_panel.map(x => x.id), 'question_panel');
+  if (p.question_panel.some(x => !x.question.trim())) throw new ContractError('question_panel: empty question');
+  const urls = p.page_types.flatMap(x => x.urls.map(u => u.normalized));
+  unique(urls, 'normalized URLs');
+  if (!urls.length) throw new ContractError('protocol requires selected public audit URLs');
+  const resources = p.auxiliary_resources ?? [];
+  unique(resources.map(x => x.normalized), 'auxiliary resources');
+  if (resources.some(x => urls.includes(x.normalized))) throw new ContractError('auxiliary resource duplicates a selected URL');
+  const requestLimit = Math.min(context.scope.max_urls ?? 50, 50);
+  if (urls.length + resources.length > requestLimit) throw new ContractError('selected pages and auxiliary resources exceed request scope');
+  for (const resource of resources) {
+    for (const url of [resource.original, resource.normalized]) {
+      try { assertUrlInScope(url, context.scope); }
+      catch { throw new ContractError(`auxiliary resource outside scope: ${url}`); }
+    }
+    if (assertUrlInScope(resource.original, context.scope).href !== resource.normalized)
+      throw new ContractError(`auxiliary resource normalized URL mismatch: ${resource.original}`);
+  }
+  for (const pageType of p.page_types) {
+    if (!pageType.urls.length) throw new ContractError(`empty page type: ${pageType.id}`);
+    for (const url of pageType.urls) {
+      for (const value of [url.original, url.normalized]) {
+        try { if (!['http:', 'https:'].includes(new URL(value).protocol)) throw new Error(); }
+        catch { throw new ContractError(`invalid HTTP URL: ${value}`); }
+      }
+      if (/\/(?:robots\.txt|sitemap[^/]*\.xml)$/i.test(decodeURIComponent(new URL(url.normalized).pathname)))
+        throw new ContractError(`${url.normalized}: robots and sitemap must be auxiliary resources`);
+      if (url.public_indexing === false) validateApplicabilityDecision(url.indexing_decision,
+        ['not_public_index_target'], `${url.normalized}.indexing_decision`);
+      else if (url.indexing_decision) throw new ContractError(`${url.normalized}: indexing decision requires public_indexing=false`);
+    }
+  }
+  const publicUrls = p.page_types.flatMap(type => type.urls.filter(url => url.public_indexing !== false).map(url => url.normalized));
+  for (const family of p.families) {
+    const definition = knownFamilies.get(family.family_id);
+    if (!definition) throw new ContractError(`unregistered family: ${family.family_id}`);
+    const requiredUrls = ['index-policy', 'search-crawl-policy', 'discovery'].includes(family.family_id) ? publicUrls : urls;
+    if (family.urls) {
+      unique(family.urls, `${family.family_id}.urls`);
+      if (definition.unit_scope === 'site' || !sameSet(family.urls, requiredUrls))
+        throw new ContractError(`${family.family_id}: use unit not_applicable decisions instead of omitting selected URLs`);
+    }
+    if (family.engines) {
+      unique(family.engines, `${family.family_id}.engines`);
+      const requiredEngines = definition.engines ?? p.engines;
+      if (definition.unit_scope !== 'engine_url' || !sameSet(family.engines, requiredEngines))
+        throw new ContractError(`${family.family_id}: target engine units cannot be omitted`);
+    }
+    const expected = family.family_id === 'fetch-response' ? true :
+      ['index-policy', 'discovery'].includes(family.family_id) ? publicUrls.length > 0 :
+      family.family_id === 'search-crawl-policy' ? publicUrls.length > 0 && p.engines.length > 0 :
+      family.family_id === 'question-coverage' ? p.question_panel.length > 0 :
+      family.family_id === 'naver-markup' ? p.engines.includes('naver') : null;
+    if (expected !== null && family.applicable !== expected)
+      throw new ContractError(`${family.family_id}: applicability contradicts fixed URL/engine conditions`);
+    if (family.applicable) {
+      if (family.applicability_decision) throw new ContractError(`${family.family_id}: decision only allowed when inapplicable`);
+    } else {
+      const decision = validateApplicabilityDecision(family.applicability_decision,
+        definition.na_conditions, `${family.family_id}.applicability_decision`);
+      const automaticCondition = family.family_id === 'search-crawl-policy' ?
+        (p.engines.length ? 'no_public_index_target' : 'no_target_engine') :
+        ['index-policy', 'discovery'].includes(family.family_id) ? 'no_public_index_target' :
+        family.family_id === 'question-coverage' ? 'no_fixed_questions' :
+        family.family_id === 'naver-markup' ? 'naver_not_target' : null;
+      if (automaticCondition && decision.condition !== automaticCondition)
+        throw new ContractError(`${family.family_id}: decision conflicts with fixed conditions`);
+    }
+    if (family.applicable && ['question-coverage', 'answer-clarity'].includes(family.family_id) && !p.question_panel.length)
+      throw new ContractError(`${family.family_id}: fixed question panel required`);
+  }
+  const sorted = { ...p,
+    ...(p.auxiliary_resources ? { auxiliary_resources: [...resources].sort((a, b) => compare(a.normalized, b.normalized)) } : {}),
+    engines: [...p.engines].sort(),
+    families: [...p.families].sort((a, b) => compare(a.family_id, b.family_id)).map(f => ({ ...f,
+      ...(f.urls ? { urls: [...f.urls].sort() } : {}),
+      ...(f.engines ? { engines: [...f.engines].sort() } : {}) })),
+    page_types: [...p.page_types].sort((a, b) => compare(a.id, b.id)).map(t => ({ ...t,
+      urls: [...t.urls].sort((a, b) => compare(a.normalized, b.normalized)).map(url => ({ ...url,
+        public_indexing: url.public_indexing !== false })) })) };
+  const protocol = stable({ ...sorted, target: context.target, environment: context.environment,
+    evaluation_version: context.evaluation_version, protocol_version: context.protocol_version });
+  const hashInput = structuredClone(protocol);
+  for (const type of hashInput.page_types) for (const url of type.urls) if (url.indexing_decision)
+    url.indexing_decision = { condition: url.indexing_decision.condition };
+  for (const family of hashInput.families) if (family.applicability_decision)
+    family.applicability_decision = { condition: family.applicability_decision.condition };
+  if (hashInput.target_identity) {
+    for (const mapping of hashInput.target_identity.native_properties ?? []) delete mapping.evidence_ref;
+    if (hashInput.target_identity.panel) delete hashInput.target_identity.panel.evidence_ref;
+  }
+  return { protocol, protocol_hash: createHash('sha256').update(JSON.stringify(stable(hashInput))).digest('hex') };
+}