@typedstandards/civic-typed-harness 0.1.0

package/dist/format/sources.js

@@ -0,0 +1,93 @@
+// Civic data-source registry (FORMAT-EXTENSION group) — the source ids the
+// civic vocabulary knows, their display names, catalog types, and endpoint
+// coordinates, plus the default tool-name → source-id resolver and the
+// display helpers. Relocated from civic-ai-tools-website:
+//   - endpoint constants + display helpers: `src/lib/evidence/data-sources.ts:15–16, 41–76`
+//   - agent titles/server URLs: `src/lib/evidence/provenance.ts:132–148`
+//   - static tool→source map: `src/lib/mcp/operation-types.ts:21–33` (the
+//     app's MCP registry itself stays app-side; this is the harness's civic
+//     DEFAULT — callers supply their own resolver to add or replace sources).
+//
+// Endpoint URLs name the demo deployment's data plane, so the registry is a
+// typed config input (config-not-constants, S2 brief §2) with the civic
+// values as the exported default map.
+/** The civic default registry — the three demo sources. */
+export const CIVIC_SOURCE_REGISTRY = {
+    socrata: {
+        displayName: 'Socrata',
+        agentTitle: 'Socrata MCP Server',
+        serverUrl: 'https://socrata-mcp.civicaitools.org',
+        catalogType: 'socrata',
+    },
+    'data-commons': {
+        displayName: 'Data Commons',
+        agentTitle: 'Google Data Commons MCP Server',
+        serverUrl: 'https://api.datacommons.org/mcp',
+        catalogType: 'data-commons',
+        aggregatePortalUrl: 'https://api.datacommons.org/mcp',
+    },
+    'boston-opencontext': {
+        displayName: 'Boston OpenContext',
+        agentTitle: 'Boston OpenContext MCP Server',
+        serverUrl: 'https://data-mcp.boston.gov/mcp',
+        catalogType: 'ckan',
+        aggregatePortalUrl: 'https://data.boston.gov',
+    },
+};
+/** The source id untagged/pre-M9.1 captures fall back to. Socrata was the
+ *  only source when those packages were written. */
+export const FALLBACK_SOURCE_ID = 'socrata';
+/** Is this source dataset-keyed (one `dataSources` entry per dataset) rather
+ *  than aggregate? Unknown ids are neither — they contribute no entry. */
+export function isDatasetKeyedSource(sourceId, registry = CIVIC_SOURCE_REGISTRY) {
+    const info = registry[sourceId];
+    return info !== undefined && info.aggregatePortalUrl === undefined;
+}
+/** Civic default tool-name → source-id map (mirrors the demo app's static
+ *  MCP-layer map; used as a fallback when a trace span carries no
+ *  `mcp.source` attribute). */
+export const CIVIC_TOOL_SOURCE_MAP = {
+    get_data: 'socrata',
+    search: 'socrata',
+    fetch: 'socrata',
+    search_indicators: 'data-commons',
+    get_observations: 'data-commons',
+    ckan__search_datasets: 'boston-opencontext',
+    ckan__get_dataset: 'boston-opencontext',
+    ckan__query_data: 'boston-opencontext',
+    ckan__get_schema: 'boston-opencontext',
+    ckan__execute_sql: 'boston-opencontext',
+    ckan__aggregate_data: 'boston-opencontext',
+};
+/** The civic default `ToolSourceResolver`. */
+export const civicToolSourceResolver = (toolName) => CIVIC_TOOL_SOURCE_MAP[toolName];
+/** Human-friendly display label for a `sourceId`. Unknown ids fall back to a
+ *  capitalised form of the raw id so new sources render sensibly before the
+ *  registry is updated. Missing ids coerce to the fallback source (pre-M9.3
+ *  packages have no `sourceId` on dataSources entries). */
+export function displayNameForSource(sourceId, registry = CIVIC_SOURCE_REGISTRY) {
+    const id = sourceId || FALLBACK_SOURCE_ID;
+    return (registry[id]?.displayName
+        ?? id
+            .split('-')
+            .filter(Boolean)
+            .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+            .join(' '));
+}
+/** Format a `dataSources` array as a compact, de-duplicated summary string
+ *  (middle-dot separated). Returns `null` when the array is empty or missing,
+ *  letting callers render a fallback. */
+export function formatDataSourcesSummary(entries, registry = CIVIC_SOURCE_REGISTRY) {
+    if (!entries || entries.length === 0)
+        return null;
+    const seen = new Set();
+    const ordered = [];
+    for (const entry of entries) {
+        const name = displayNameForSource(entry.sourceId, registry);
+        if (seen.has(name))
+            continue;
+        seen.add(name);
+        ordered.push(name);
+    }
+    return ordered.join(' · ');
+}

package/dist/format/vocabulary.d.ts

@@ -0,0 +1,37 @@
+/** The `civic:` JSON-LD namespace URI. Part of the civic vocabulary itself
+ *  (it names the term set, not a deployment), so it is a constant, not a
+ *  config input. */
+export declare const CIVIC_NS = "https://civicaitools.org/ns/evidence/";
+/** The civic evidence-package JSON-LD `@context`: prov / xsd / civic /
+ *  dcterms, in the reference emission order (legacy-chain byte discipline —
+ *  insertion order is the byte contract). */
+export declare function makeCivicProvContext(): Record<string, string>;
+/** Root of the `urn:civic-evidence:` id scheme. */
+export declare const CIVIC_URN_PREFIX = "urn:civic-evidence";
+/** Package-scoped node id: `urn:civic-evidence:<packageId>:<type>:<id>`. */
+export declare function civicUrn(packageId: string, type: string, id: string): string;
+/** Model-agent id: `urn:civic-evidence:model:<modelId>` (slashes in the
+ *  model identifier collapse to dashes). */
+export declare function civicModelUrn(model: string): string;
+/** MCP source-agent id: `urn:civic-evidence:mcp-server:<sourceId>`. Known
+ *  source ids are emitted raw (they are already URN-safe by construction);
+ *  callers emitting an UNKNOWN id must encode it first (the capture-side
+ *  builder uses `encodeURIComponent`, matching the reference behavior). */
+export declare function civicSourceAgentUrn(sourceId: string): string;
+/** Platform-agent id: `urn:civic-evidence:platform:<platformId>`. */
+export declare function civicPlatformUrn(platformId: string): string;
+/**
+ * Platform-agent identity — WHO published, as a PROV agent. A typed config
+ * input (config-not-constants, S2 brief §2): every deployment names its own;
+ * the civicaitools.org demo values are the exported default below.
+ */
+export interface PlatformAgentConfig {
+    /** Stable platform id — becomes `urn:civic-evidence:platform:<id>`. */
+    id: string;
+    /** Human-readable platform title (`dcterms:title`). */
+    title: string;
+    /** Public URL of the deployment (`civic:url`). */
+    url: string;
+}
+/** Demo default: the civicaitools.org reference deployment's identity. */
+export declare const CIVICAITOOLS_PLATFORM_AGENT: PlatformAgentConfig;

package/dist/format/vocabulary.js

@@ -0,0 +1,53 @@
+// Civic PROV-O vocabulary (FORMAT-EXTENSION group) — the `civic:` JSON-LD
+// namespace, the `urn:civic-evidence:` id scheme, and the platform-agent
+// identity shape. Relocated from civic-ai-tools-website
+// `src/lib/evidence/provenance.ts` (`:51` urn scheme, `:179–184` platform
+// agent, `:384–391` context) per the S2 brief §1.
+//
+// This module defines vocabulary; it never walks a trace. The capture-side
+// provenance BUILDER (src/capture/provenance.ts) imports its terms from here —
+// that direction is the package's internal module boundary.
+import { makeProvContext, PROV_NS, XSD_NS, DCTERMS_NS } from '@typedstandards/produce-core';
+/** The `civic:` JSON-LD namespace URI. Part of the civic vocabulary itself
+ *  (it names the term set, not a deployment), so it is a constant, not a
+ *  config input. */
+export const CIVIC_NS = 'https://civicaitools.org/ns/evidence/';
+/** The civic evidence-package JSON-LD `@context`: prov / xsd / civic /
+ *  dcterms, in the reference emission order (legacy-chain byte discipline —
+ *  insertion order is the byte contract). */
+export function makeCivicProvContext() {
+    return makeProvContext({
+        prov: PROV_NS,
+        xsd: XSD_NS,
+        civic: CIVIC_NS,
+        dcterms: DCTERMS_NS,
+    });
+}
+/** Root of the `urn:civic-evidence:` id scheme. */
+export const CIVIC_URN_PREFIX = 'urn:civic-evidence';
+/** Package-scoped node id: `urn:civic-evidence:<packageId>:<type>:<id>`. */
+export function civicUrn(packageId, type, id) {
+    return `${CIVIC_URN_PREFIX}:${packageId}:${type}:${id}`;
+}
+/** Model-agent id: `urn:civic-evidence:model:<modelId>` (slashes in the
+ *  model identifier collapse to dashes). */
+export function civicModelUrn(model) {
+    return `${CIVIC_URN_PREFIX}:model:${model.replace(/\//g, '-')}`;
+}
+/** MCP source-agent id: `urn:civic-evidence:mcp-server:<sourceId>`. Known
+ *  source ids are emitted raw (they are already URN-safe by construction);
+ *  callers emitting an UNKNOWN id must encode it first (the capture-side
+ *  builder uses `encodeURIComponent`, matching the reference behavior). */
+export function civicSourceAgentUrn(sourceId) {
+    return `${CIVIC_URN_PREFIX}:mcp-server:${sourceId}`;
+}
+/** Platform-agent id: `urn:civic-evidence:platform:<platformId>`. */
+export function civicPlatformUrn(platformId) {
+    return `${CIVIC_URN_PREFIX}:platform:${platformId}`;
+}
+/** Demo default: the civicaitools.org reference deployment's identity. */
+export const CIVICAITOOLS_PLATFORM_AGENT = {
+    id: 'civic-ai-tools',
+    title: 'Civic AI Tools',
+    url: 'https://civicaitools.org',
+};

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export * from './format/vocabulary.ts';
+export * from './format/sources.ts';
+export * from './format/dathere.ts';
+export * from './format/profiles.ts';
+export * from './capture/trace.ts';
+export * from './capture/skill-metadata.ts';
+export * from './capture/data-sources.ts';
+export * from './capture/provenance.ts';
+export * from './rubric/adversarial-eval-core.ts';

package/dist/index.js

@@ -0,0 +1,30 @@
+// civic-typed-harness — the civic DOMAIN layer of ADR-0021's format/domain
+// line, relocated from the reference app (civic-ai-tools-website) per the S2
+// brief. The operating rule: THE HARNESS DERIVES, THE CORE ASSEMBLES — every
+// export here produces domain-derived values (vocabulary terms, datHere
+// policy fields, provenance graphs, capture artifacts) that feed
+// @typedstandards/produce-core's neutral envelope assembly.
+//
+// Internal module boundary (structure-for-the-future; see README):
+//   - format/  — the civic FORMAT-EXTENSION: what extends the standard
+//     (civic: vocabulary, source registry, datHere policy, profile constants).
+//   - capture/ — what PRODUCES under it: TraceBuilder, skill-metadata
+//     extraction, data-source population, and the provenance BUILDER (which
+//     imports its vocabulary from format/). Clock + RNG live here only.
+//   - rubric/  — the adversarial-evaluation pure core (Q26-pinned hash).
+//
+// Purity contract (harness-grade): no I/O and no environment reads anywhere;
+// clock + RNG permitted in capture modules only and injectable for tests.
+// Enforced by eslint.config.mjs and src/purity.test.ts.
+// Format-extension group
+export * from "./format/vocabulary.js";
+export * from "./format/sources.js";
+export * from "./format/dathere.js";
+export * from "./format/profiles.js";
+// Capture group
+export * from "./capture/trace.js";
+export * from "./capture/skill-metadata.js";
+export * from "./capture/data-sources.js";
+export * from "./capture/provenance.js";
+// Rubric group
+export * from "./rubric/adversarial-eval-core.js";

package/dist/rubric/adversarial-eval-core.d.ts

@@ -0,0 +1,27 @@
+import { type EvidencePackage, type EvaluationResults } from '@typedstandards/produce-core';
+/** Rubric identifier carried as the `scoringRubric` payload field and the
+ *  methodology `testSet`. Six criteria, 1-10 each, plus aggregate. */
+export declare const RUBRIC_ID = "civicaitools-adversarial-rubric/six-criterion-v1";
+export declare const EVALUATION_CRITERIA: readonly ["dataSourceIdentification", "quantitativeClaimSupport", "confoundersAndBias", "geographicScope", "limitationsNoted", "contradictoryConclusion"];
+export declare const EVALUATION_RUBRIC = "You are an independent evaluator assessing an AI-generated civic data analysis.\n\nYou will receive:\n1. The original prompt (question asked)\n2. The tool calls made (MCP queries to Socrata open data portals)\n3. The AI's final output/analysis\n\nEvaluate the analysis against these 6 criteria, scoring each 1-10:\n\na) **Data Source Identification** (1-10): Does the output correctly identify the data source(s) and time period? Are dataset IDs, portal domains, and date ranges accurate?\n\nb) **Quantitative Claim Support** (1-10): Are the quantitative claims (numbers, percentages, rankings) supported by the data returned in the tool calls? Cross-check key figures against the raw data.\n\nc) **Confounders and Bias** (1-10): Does the analysis acknowledge obvious confounders, selection biases, or framing issues? Are there lurking variables or cherry-picked timeframes?\n\nd) **Geographic Scope** (1-10): Is the geographic scope appropriate for the question? Does the analysis avoid over-generalizing from one jurisdiction?\n\ne) **Limitations Noted** (1-10): Are limitations and caveats noted? Does the analysis flag data quality issues, missing fields, or incomplete coverage?\n\nf) **Contradictory Conclusion** (1-10): Could the same data reasonably support a contradictory conclusion? Does the analysis consider alternative interpretations?\n\nRespond in this exact JSON format (no markdown fences, just raw JSON):\n{\n  \"dataSourceIdentification\": { \"score\": <1-10>, \"comment\": \"<1-2 sentences>\" },\n  \"quantitativeClaimSupport\": { \"score\": <1-10>, \"comment\": \"<1-2 sentences>\" },\n  \"confoundersAndBias\": { \"score\": <1-10>, \"comment\": \"<1-2 sentences>\" },\n  \"geographicScope\": { \"score\": <1-10>, \"comment\": \"<1-2 sentences>\" },\n  \"limitationsNoted\": { \"score\": <1-10>, \"comment\": \"<1-2 sentences>\" },\n  \"contradictoryConclusion\": { \"score\": <1-10>, \"comment\": \"<1-2 sentences>\" },\n  \"overallScore\": <average of all 6 scores, one decimal>,\n  \"assessment\": \"<2-4 sentence overall assessment>\"\n}";
+/** The methodology's `promptSetVersion`: SHA-256 of the rubric text, computed
+ *  once at module load. Any wording change to the rubric produces a new
+ *  version, so an attestation pins the exact prompt set that scored it. */
+export declare const RUBRIC_VERSION_SHA256: string;
+/** Build the user-turn evaluation content from a package's signed fields. */
+export declare function buildEvaluationPrompt(pkg: EvidencePackage): string;
+export type ParsedEvaluation = {
+    ok: true;
+    results: EvaluationResults;
+} | {
+    ok: false;
+    error: string;
+    raw: string;
+};
+/**
+ * Parse + validate an evaluator response into structured results. Pure —
+ * unit-tested directly. Strips markdown fences (some models wrap despite the
+ * instruction), requires every criterion with a numeric score, and recomputes
+ * the aggregate when the model omits it.
+ */
+export declare function parseEvaluationResponse(raw: string): ParsedEvaluation;

package/dist/rubric/adversarial-eval-core.js

@@ -0,0 +1,128 @@
+// Adversarial evaluation — PURE core (RUBRIC group): the civic six-criterion
+// rubric text, its pinned identifiers, the prompt builder, and the response
+// parser. Relocated from civic-ai-tools-website
+// `src/lib/evidence/adversarial-eval-core.ts:13, 24–55, 66–94` per the S2
+// brief §1. The model runner and DB/blob emission (`adversarial-eval.ts`)
+// stay app-side — this module never performs I/O.
+//
+// ⚠ RUBRIC_VERSION_SHA256 is a Q26-pinned version hash: attestations pin the
+// exact prompt set that scored them by this value. The rubric text below is
+// byte-exact with the reference implementation (the relocation changes no
+// hashes), and the test suite asserts the literal digest. Any wording change
+// to EVALUATION_RUBRIC is a NEW rubric version — never edit the text without
+// versioning intent.
+import { sha256Hex, } from '@typedstandards/produce-core';
+/** Rubric identifier carried as the `scoringRubric` payload field and the
+ *  methodology `testSet`. Six criteria, 1-10 each, plus aggregate. */
+export const RUBRIC_ID = 'civicaitools-adversarial-rubric/six-criterion-v1';
+export const EVALUATION_CRITERIA = [
+    'dataSourceIdentification',
+    'quantitativeClaimSupport',
+    'confoundersAndBias',
+    'geographicScope',
+    'limitationsNoted',
+    'contradictoryConclusion',
+];
+export const EVALUATION_RUBRIC = `You are an independent evaluator assessing an AI-generated civic data analysis.
+
+You will receive:
+1. The original prompt (question asked)
+2. The tool calls made (MCP queries to Socrata open data portals)
+3. The AI's final output/analysis
+
+Evaluate the analysis against these 6 criteria, scoring each 1-10:
+
+a) **Data Source Identification** (1-10): Does the output correctly identify the data source(s) and time period? Are dataset IDs, portal domains, and date ranges accurate?
+
+b) **Quantitative Claim Support** (1-10): Are the quantitative claims (numbers, percentages, rankings) supported by the data returned in the tool calls? Cross-check key figures against the raw data.
+
+c) **Confounders and Bias** (1-10): Does the analysis acknowledge obvious confounders, selection biases, or framing issues? Are there lurking variables or cherry-picked timeframes?
+
+d) **Geographic Scope** (1-10): Is the geographic scope appropriate for the question? Does the analysis avoid over-generalizing from one jurisdiction?
+
+e) **Limitations Noted** (1-10): Are limitations and caveats noted? Does the analysis flag data quality issues, missing fields, or incomplete coverage?
+
+f) **Contradictory Conclusion** (1-10): Could the same data reasonably support a contradictory conclusion? Does the analysis consider alternative interpretations?
+
+Respond in this exact JSON format (no markdown fences, just raw JSON):
+{
+  "dataSourceIdentification": { "score": <1-10>, "comment": "<1-2 sentences>" },
+  "quantitativeClaimSupport": { "score": <1-10>, "comment": "<1-2 sentences>" },
+  "confoundersAndBias": { "score": <1-10>, "comment": "<1-2 sentences>" },
+  "geographicScope": { "score": <1-10>, "comment": "<1-2 sentences>" },
+  "limitationsNoted": { "score": <1-10>, "comment": "<1-2 sentences>" },
+  "contradictoryConclusion": { "score": <1-10>, "comment": "<1-2 sentences>" },
+  "overallScore": <average of all 6 scores, one decimal>,
+  "assessment": "<2-4 sentence overall assessment>"
+}`;
+/** The methodology's `promptSetVersion`: SHA-256 of the rubric text, computed
+ *  once at module load. Any wording change to the rubric produces a new
+ *  version, so an attestation pins the exact prompt set that scored it. */
+export const RUBRIC_VERSION_SHA256 = sha256Hex(EVALUATION_RUBRIC);
+/** Build the user-turn evaluation content from a package's signed fields. */
+export function buildEvaluationPrompt(pkg) {
+    const toolCallSummary = pkg.queries
+        .map((q, i) => {
+        const argStr = Object.entries(q.arguments)
+            .map(([k, v]) => `${k}=${JSON.stringify(v)}`)
+            .join(', ');
+        return `  ${i + 1}. ${q.tool}(${argStr}) → ${q.resultRows ?? '?'} rows`;
+    })
+        .join('\n');
+    const dataSources = pkg.dataSources
+        .map(ds => `  - ${ds.datasetUrl} (accessed ${ds.accessTimestamp})`)
+        .join('\n');
+    return `## Original Prompt
+${pkg.prompt.text || '[prompt text not available]'}
+
+## Tool Calls Made (${pkg.queries.length} total)
+${toolCallSummary || '  (none)'}
+
+## Data Sources
+${dataSources || '  (none)'}
+
+## Model Used
+${pkg.cost.model}
+
+## AI Output
+${pkg.output}`;
+}
+/**
+ * Parse + validate an evaluator response into structured results. Pure —
+ * unit-tested directly. Strips markdown fences (some models wrap despite the
+ * instruction), requires every criterion with a numeric score, and recomputes
+ * the aggregate when the model omits it.
+ */
+export function parseEvaluationResponse(raw) {
+    const jsonStr = raw.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
+    let parsed;
+    try {
+        parsed = JSON.parse(jsonStr);
+    }
+    catch {
+        return { ok: false, error: 'Evaluator returned invalid JSON', raw };
+    }
+    const perCriterion = {};
+    for (const key of EVALUATION_CRITERIA) {
+        const entry = parsed[key];
+        if (!entry || typeof entry.score !== 'number') {
+            return { ok: false, error: `Missing or invalid rubric criterion: ${key}`, raw };
+        }
+        perCriterion[key] = {
+            score: entry.score,
+            comment: typeof entry.comment === 'string' ? entry.comment : '',
+        };
+    }
+    const overallScore = typeof parsed.overallScore === 'number'
+        ? parsed.overallScore
+        : EVALUATION_CRITERIA.reduce((sum, k) => sum + perCriterion[k].score, 0) /
+            EVALUATION_CRITERIA.length;
+    return {
+        ok: true,
+        results: {
+            perCriterion,
+            overallScore,
+            assessment: typeof parsed.assessment === 'string' ? parsed.assessment : '',
+        },
+    };
+}

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@typedstandards/civic-typed-harness",
+  "version": "0.1.0",
+  "description": "Civic-domain harness for Typed Standards evidence packages — the DOMAIN layer of the ADR-0021 format/domain line: civic PROV-O vocabulary and provenance builder, datHere envelope-policy derivations, and the capture machinery (trace builder, skill-metadata extraction, data-source population, adversarial rubric core). Derives; @typedstandards/produce-core assembles.",
+  "license": "MIT",
+  "author": "Nathan Storey",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "clean": "rm -rf dist tsconfig.tsbuildinfo",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "lint": "eslint .",
+    "test": "node --test --experimental-strip-types \"src/**/*.test.ts\"",
+    "prepare": "npm run build",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "dependencies": {
+    "@typedstandards/produce-core": "^0.2.0"
+  },
+  "devDependencies": {
+    "@typescript-eslint/parser": "^8",
+    "eslint": "^9",
+    "typescript": "^5.9.3"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/npstorey/civic-ai-tools.git",
+    "directory": "packages/civic-typed-harness"
+  },
+  "homepage": "https://github.com/npstorey/civic-ai-tools/tree/main/packages/civic-typed-harness#readme",
+  "bugs": {
+    "url": "https://github.com/npstorey/civic-ai-tools/issues"
+  },
+  "keywords": [
+    "typed-standards",
+    "evidence",
+    "provenance",
+    "civic",
+    "harness",
+    "prov-o",
+    "otel"
+  ]
+}