@aperant/framework 0.8.1 → 0.8.6

package/src/cli/verify-proof/manifest-validator.mjs

@@ -0,0 +1,184 @@
+/**
+ * manifest-validator.mjs — hand-rolled JSON-Schema subset validator.
+ *
+ * Validates driver manifests against manifest-schema.json. Implements
+ * exactly the JSON-Schema keywords we need:
+ *
+ *   type / required / properties / items / enum / const / pattern / oneOf
+ *
+ * Returns { valid: true } or { valid: false, errors: [{ path, message }] }
+ * where `path` is a JSON-pointer (`/transport/local_command/binary`).
+ *
+ * No external deps — keeps the validator installable as `dist/` with the
+ * framework package and avoids dragging ajv (or any v5+ JSON-Schema
+ * validator) into the runtime surface for ~80 lines of validator code per
+ * AC15. We don't need allOf / anyOf / definitions — manifest-schema.json
+ * is intentionally a thin contract.
+ */
+
+import { readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+/** Lazy-loaded JSON-Schema (read once on first call). */
+let cachedSchema = null
+
+/** Read the bundled manifest-schema.json. */
+export function loadManifestSchema() {
+	if (cachedSchema !== null) return cachedSchema
+	const schemaPath = join(__dirname, 'manifest-schema.json')
+	cachedSchema = JSON.parse(readFileSync(schemaPath, 'utf-8'))
+	return cachedSchema
+}
+
+/**
+ * Validate a manifest object against the bundled schema.
+ *
+ * @param {unknown} manifest — usually JSON.parse(manifest.json contents).
+ * @param {object} [schema] — override the bundled schema (used by tests).
+ * @returns {{valid: true} | {valid: false, errors: Array<{path: string, message: string}>}}
+ */
+export function validateManifest(manifest, schema) {
+	const root = schema ?? loadManifestSchema()
+	const errors = []
+	validateNode(manifest, root, '', errors)
+	if (errors.length === 0) return { valid: true }
+	return { valid: false, errors }
+}
+
+/** Type check — JSON-Schema's union of "string", "number", "object", "array", "boolean", "null". */
+function checkType(value, type) {
+	if (type === 'string') return typeof value === 'string'
+	if (type === 'number') return typeof value === 'number' && !Number.isNaN(value)
+	if (type === 'integer') return typeof value === 'number' && Number.isInteger(value)
+	if (type === 'boolean') return typeof value === 'boolean'
+	if (type === 'null') return value === null
+	if (type === 'array') return Array.isArray(value)
+	if (type === 'object') {
+		return value !== null && typeof value === 'object' && !Array.isArray(value)
+	}
+	return false
+}
+
+/**
+ * Human-readable name for the actual shape of a value. Distinguishes
+ * `null`, `array`, and plain `object` — JS `typeof` reports all three as
+ * `'object'`, which makes error messages like "expected type object, got
+ * object" misleading.
+ *
+ * @param {unknown} value
+ * @returns {string}
+ */
+function describeType(value) {
+	if (value === null) return 'null'
+	if (Array.isArray(value)) return 'array'
+	return typeof value
+}
+
+/**
+ * Recursive validator. `errors` is mutated.
+ *
+ * @param {unknown} value
+ * @param {object} schema
+ * @param {string} path — JSON-pointer
+ * @param {Array<{path: string, message: string}>} errors
+ */
+function validateNode(value, schema, path, errors) {
+	// const
+	if ('const' in schema) {
+		if (value !== schema.const) {
+			errors.push({
+				path: path || '/',
+				message: `expected const ${JSON.stringify(schema.const)}, got ${JSON.stringify(value)}`,
+			})
+			return
+		}
+	}
+
+	// type
+	if (schema.type !== undefined) {
+		const types = Array.isArray(schema.type) ? schema.type : [schema.type]
+		if (!types.some((t) => checkType(value, t))) {
+			errors.push({
+				path: path || '/',
+				message: `expected type ${types.join('|')}, got ${describeType(value)}`,
+			})
+			return
+		}
+	}
+
+	// enum
+	if (Array.isArray(schema.enum)) {
+		if (!schema.enum.includes(value)) {
+			errors.push({
+				path: path || '/',
+				message: `value ${JSON.stringify(value)} not in enum [${schema.enum.join(', ')}]`,
+			})
+			return
+		}
+	}
+
+	// pattern
+	if (typeof schema.pattern === 'string' && typeof value === 'string') {
+		const re = new RegExp(schema.pattern)
+		if (!re.test(value)) {
+			errors.push({
+				path: path || '/',
+				message: `value ${JSON.stringify(value)} does not match pattern ${schema.pattern}`,
+			})
+			return
+		}
+	}
+
+	// oneOf
+	if (Array.isArray(schema.oneOf)) {
+		let matchCount = 0
+		const branchErrors = []
+		for (let i = 0; i < schema.oneOf.length; i++) {
+			const sub = schema.oneOf[i]
+			const subErrors = []
+			validateNode(value, sub, path, subErrors)
+			if (subErrors.length === 0) matchCount++
+			else branchErrors.push({ branch: i, errors: subErrors })
+		}
+		if (matchCount !== 1) {
+			errors.push({
+				path: path || '/',
+				message:
+					matchCount === 0
+						? `oneOf: value matched no branch (${branchErrors.length} branches tried)`
+						: `oneOf: value matched ${matchCount} branches (must match exactly 1)`,
+			})
+			return
+		}
+	}
+
+	// object validation
+	if (checkType(value, 'object')) {
+		const obj = /** @type {Record<string, unknown>} */ (value)
+		if (Array.isArray(schema.required)) {
+			for (const key of schema.required) {
+				if (!(key in obj)) {
+					errors.push({ path: `${path}/${key}`, message: `required property "${key}" missing` })
+				}
+			}
+		}
+		if (schema.properties && typeof schema.properties === 'object') {
+			for (const [key, subSchema] of Object.entries(schema.properties)) {
+				if (key in obj) {
+					validateNode(obj[key], /** @type {object} */ (subSchema), `${path}/${key}`, errors)
+				}
+			}
+		}
+	}
+
+	// array validation
+	if (checkType(value, 'array') && schema.items) {
+		const arr = /** @type {unknown[]} */ (value)
+		for (let i = 0; i < arr.length; i++) {
+			validateNode(arr[i], schema.items, `${path}/${i}`, errors)
+		}
+	}
+}

package/src/cli/verify-proof/resolver.mjs

@@ -0,0 +1,330 @@
+/**
+ * resolver.mjs — driver selection + fallback chain.
+ *
+ * Given a project's runtime capabilities, the discovered drivers, and the
+ * test's required IDL capabilities, return the best driver — or throw
+ * UnsatisfiedRequiredCapabilityError if no driver in the chain satisfies
+ * a REQUIRED capability (ID-02 fail-closed).
+ *
+ * Scoring (ID-02, ordered high → low):
+ *   1. capability_coverage — how many required capabilities the driver has
+ *   2. compatibility — does the runtime match the manifest.targets?
+ *   3. stability — ga > beta > experimental
+ *   4. priority — higher wins (manifest.priority)
+ *   5. locality — bundled (packages/framework/drivers/) > user-installed
+ *      (.aperant/drivers/) > registry (.aperant/registry-cache/)
+ *
+ * Ties are broken by the next dimension; if every dimension is equal the
+ * driver with the lexicographically smaller driverId wins (deterministic).
+ *
+ * v0.8.6 — `resolveDriversForOutcomes(opts)` adds per-outcome driver
+ * dispatch for cross-surface specs (per spec §ID-03). It accepts the
+ * positional `outcomes[]` produced by `parseUserOutcomes` and returns
+ * `Array<{outcome_id, surface, driver}>` — one entry per
+ * (outcome, surface) pair. For `[electron+web]` outcomes it emits TWO
+ * entries with the same outcome_id and different drivers. The
+ * outcome_id field is passed through verbatim from parser → resolver
+ * → workflow runner with no name rewriting (the positional-ID
+ * contract).
+ *
+ * Existing `resolveDriver()` and `scoreDriver()` exports are unchanged.
+ */
+
+import { UnsatisfiedRequiredCapabilityError } from '@aperant/framework/driver-sdk'
+
+/**
+ * @typedef {import('@aperant/framework/driver-sdk').DriverManifest} DriverManifest
+ * @typedef {import('./runtime-detect.mjs').RuntimeCapabilities} RuntimeCapabilities
+ */
+
+const STABILITY_RANK = { ga: 3, beta: 2, experimental: 1 }
+const LOCALITY_RANK = { bundled: 3, user: 2, registry: 1 }
+
+/**
+ * Score a single driver against the project's runtime + required capabilities.
+ *
+ * @param {DriverManifest & {locality?: 'bundled'|'user'|'registry'}} manifest
+ * @param {RuntimeCapabilities} runtime
+ * @param {string[]} required
+ * @returns {{score: number[], reason?: string}}
+ */
+export function scoreDriver(manifest, runtime, required) {
+	const caps = new Set(manifest.capabilities ?? [])
+	const coverage = required.filter((c) => caps.has(c)).length
+	const compatibility = matchesRuntime(manifest, runtime) ? 1 : 0
+	const stability = STABILITY_RANK[manifest.stability] ?? 0
+	const priority = manifest.priority ?? 0
+	const locality = LOCALITY_RANK[manifest.locality ?? 'bundled'] ?? 0
+	return { score: [coverage, compatibility, stability, priority, locality] }
+}
+
+/**
+ * Whether a manifest's targets[] mentions a runtime the project has.
+ *
+ * @param {DriverManifest} manifest
+ * @param {RuntimeCapabilities} runtime
+ * @returns {boolean}
+ */
+function matchesRuntime(manifest, runtime) {
+	const targets = manifest.targets ?? []
+	for (const t of targets) {
+		// Strip any version range suffix (e.g. "electron@>=25" → "electron").
+		const stripped = String(t).split('@')[0].toLowerCase()
+		if (stripped === 'electron' && runtime.is_electron) return true
+		if (stripped === 'tauri' && runtime.is_tauri) return true
+		if ((stripped === 'browser' || stripped === 'web') && runtime.is_web_frontend) return true
+		if (stripped === 'next' && runtime.is_nextjs) return true
+		if (stripped === 'nuxt' && runtime.is_nuxt) return true
+		if (stripped === 'expo' && runtime.is_expo) return true
+		if (stripped === 'react-native' && runtime.is_react_native) return true
+		if (stripped === 'node' && runtime.has_node) return true
+		if (stripped === 'cli' && runtime.has_node) return true
+		if (stripped === 'api' && runtime.has_node) return true
+		if (stripped === 'fake') return true // fixture-only — always matches
+	}
+	return false
+}
+
+/**
+ * Compare two driver-score tuples lexicographically.
+ * @param {number[]} a
+ * @param {number[]} b
+ * @returns {number}
+ */
+function compareScores(a, b) {
+	for (let i = 0; i < Math.min(a.length, b.length); i++) {
+		if (a[i] !== b[i]) return b[i] - a[i] // higher wins
+	}
+	return 0
+}
+
+/**
+ * Resolve the best driver for a given (runtime, drivers, override, required)
+ * tuple.
+ *
+ * @param {Object} opts
+ * @param {DriverManifest[]} opts.drivers — manifests of every discovered driver.
+ * @param {RuntimeCapabilities} opts.runtime — project's runtime capabilities.
+ * @param {string[]} opts.required_capabilities — IDL verbs the test needs.
+ * @param {string[]} [opts.required_stability] — accepted stability values; default ['ga'].
+ * @param {string[]} [opts.override] — config.json:verification.runtimes[]: explicit driverId precedence.
+ * @returns {{selected: DriverManifest, fallback_attempts: Array<{driver_id: string, reason: string}>}}
+ * @throws {UnsatisfiedRequiredCapabilityError} when no driver satisfies a REQUIRED capability.
+ */
+export function resolveDriver(opts) {
+	const drivers = opts.drivers ?? []
+	const runtime = opts.runtime
+	const required = opts.required_capabilities ?? []
+	const stabilityFilter = new Set(opts.required_stability ?? ['ga'])
+	const override = opts.override ?? []
+
+	const fallback = []
+
+	// Apply override precedence first (config-listed driverIds come first).
+	const overrideOrdered = []
+	const overrideSet = new Set(override)
+	for (const id of override) {
+		const d = drivers.find((m) => m.driverId === id)
+		if (d) overrideOrdered.push(d)
+	}
+	const remaining = drivers.filter((m) => !overrideSet.has(m.driverId))
+	const ordered = [...overrideOrdered, ...remaining]
+
+	// Filter by stability; rejected drivers go into fallback_attempts.
+	const stableOnly = []
+	for (const d of ordered) {
+		if (!stabilityFilter.has(d.stability)) {
+			fallback.push({
+				driver_id: d.driverId,
+				reason: `stability ${d.stability} not in allowed set [${[...stabilityFilter].join(', ')}]`,
+			})
+			continue
+		}
+		stableOnly.push(d)
+	}
+
+	// Override is hard precedence (per opts.override docstring "explicit
+	// driverId precedence"): when set, try override-listed drivers in
+	// override order — the first one that satisfies every required
+	// capability wins, regardless of compatibility/priority/stability
+	// ranking. Non-matching override entries land in fallback. Only when
+	// every override entry misses a required cap do we fall through to
+	// score-based pick on the remaining drivers.
+	const overrideListed = stableOnly.filter((m) => overrideSet.has(m.driverId))
+	for (const candidate of overrideListed) {
+		const caps = new Set(candidate.capabilities ?? [])
+		const missing = required.filter((c) => !caps.has(c))
+		if (missing.length === 0) {
+			return { selected: candidate, fallback_attempts: fallback }
+		}
+		fallback.push({
+			driver_id: candidate.driverId,
+			reason: `override-listed but missing required capabilities: ${missing.join(', ')}`,
+		})
+	}
+
+	// Score every non-override survivor against required capabilities.
+	const nonOverride = stableOnly.filter((m) => !overrideSet.has(m.driverId))
+	const scored = nonOverride.map((m) => ({ manifest: m, ...scoreDriver(m, runtime, required) }))
+
+	// Sort highest-first, tie-break on driverId for determinism.
+	scored.sort((a, b) => {
+		const cmp = compareScores(a.score, b.score)
+		if (cmp !== 0) return cmp
+		return a.manifest.driverId.localeCompare(b.manifest.driverId)
+	})
+
+	// Pick the top scorer that covers EVERY required capability. Others go into fallback.
+	for (const candidate of scored) {
+		const caps = new Set(candidate.manifest.capabilities ?? [])
+		const missing = required.filter((c) => !caps.has(c))
+		if (missing.length === 0) {
+			return { selected: candidate.manifest, fallback_attempts: fallback }
+		}
+		fallback.push({
+			driver_id: candidate.manifest.driverId,
+			reason: `missing required capabilities: ${missing.join(', ')}`,
+		})
+	}
+
+	// Exhausted the chain — fail closed (ID-02).
+	throw new UnsatisfiedRequiredCapabilityError(
+		`No driver in the chain satisfies every required capability: [${required.join(', ')}]`,
+		required[0],
+		fallback,
+	)
+}
+
+/**
+ * Map a User Outcomes surface tag to the runtime-capability flag a driver
+ * must carry (via `matchesRuntime`) to be eligible for that surface.
+ *
+ * Closed surface set: `electron | web | cli | api | electron+web`.
+ * `electron+web` is NOT a single flag — it's a cross-surface tag the
+ * workflow runner expands into two single-surface dispatches.
+ *
+ * @param {('electron'|'web'|'cli'|'api')} surface
+ * @returns {('is_electron'|'is_web_frontend'|'has_node')}
+ */
+export function surfaceToRequiredFlag(surface) {
+	switch (surface) {
+		case 'electron':
+			return 'is_electron'
+		case 'web':
+			return 'is_web_frontend'
+		case 'cli':
+		case 'api':
+			return 'has_node'
+		default:
+			throw new Error(`surfaceToRequiredFlag: unknown surface "${surface}"`)
+	}
+}
+
+/**
+ * Resolve a driver per outcome surface tag, NOT per verify-proof run
+ * (per spec §ID-03). For each input outcome the function returns one
+ * `{outcome_id, surface, driver}` entry; for `[electron+web]` outcomes
+ * it returns TWO entries (electron + web) sharing the same outcome_id.
+ *
+ * The `outcome_id` field on each returned entry is the positional ID
+ * produced by `parseUserOutcomes` (e.g. `O1`, `O2`). The resolver does
+ * NOT rewrite or regenerate IDs — they pass through verbatim so the
+ * workflow runner can correlate proof artifacts to spec lines.
+ *
+ * Each per-surface dispatch synthesizes a runtime-capabilities view
+ * that ONLY claims the surface's required flag (e.g. `is_electron`
+ * alone for an electron-surface outcome, even when the project also
+ * carries `is_web_frontend`). This forces the existing scoring to
+ * pick the per-surface driver — without it, a project with BOTH
+ * electron AND web capabilities would always select the highest-
+ * priority bundled driver (currently `browser`) for every outcome.
+ *
+ * Delegates to the existing `resolveDriver()` for the actual scoring
+ * — the scoring code stays in one place.
+ *
+ * @param {Object} opts
+ * @param {DriverManifest[]} opts.drivers
+ * @param {RuntimeCapabilities} opts.runtime
+ * @param {Array<{id: string, surface: 'electron'|'web'|'cli'|'api'|'electron+web'}>} opts.outcomes
+ * @param {string[]} [opts.override]
+ * @param {string[]} [opts.required_stability]
+ * @param {string[]} [opts.required_capabilities] — defaults to ['screenshot']
+ * @returns {Array<{outcome_id: string, surface: string, driver: DriverManifest, fallback_attempts: Array<{driver_id: string, reason: string}>}>}
+ */
+export function resolveDriversForOutcomes(opts) {
+	const drivers = opts.drivers ?? []
+	const runtime = opts.runtime
+	const outcomes = opts.outcomes ?? []
+	const required = opts.required_capabilities ?? ['screenshot']
+	const override = opts.override
+	const stability = opts.required_stability
+
+	/** @type {Array<{outcome_id: string, surface: string, driver: DriverManifest, fallback_attempts: Array<{driver_id: string, reason: string}>}>} */
+	const results = []
+
+	for (const outcome of outcomes) {
+		const surfacesToDispatch =
+			outcome.surface === 'electron+web' ? ['electron', 'web'] : [outcome.surface]
+
+		for (const surface of surfacesToDispatch) {
+			const flag = surfaceToRequiredFlag(/** @type {'electron'|'web'|'cli'|'api'} */ (surface))
+			// Synthesize a per-surface runtime view: only the surface's
+			// required flag is on; every other flag is masked off so the
+			// scoring prefers per-surface drivers (otherwise, on a
+			// multi-runtime project, the highest-priority bundled driver
+			// always wins). `has_node` stays on for cli/api since multiple
+			// drivers may target node.
+			const surfaceRuntime = maskRuntimeForSurface(runtime, flag)
+			const resolved = resolveDriver({
+				drivers,
+				runtime: surfaceRuntime,
+				required_capabilities: required,
+				...(override ? { override } : {}),
+				...(stability ? { required_stability: stability } : {}),
+			})
+			results.push({
+				outcome_id: outcome.id,
+				surface,
+				driver: resolved.selected,
+				fallback_attempts: resolved.fallback_attempts,
+			})
+		}
+	}
+
+	return results
+}
+
+/**
+ * Return a copy of `runtime` with every flag false EXCEPT the one named
+ * by `flag`. Used to scope the resolver's compatibility check to a
+ * single surface during per-outcome dispatch.
+ *
+ * @param {RuntimeCapabilities} runtime
+ * @param {string} flag — one of: is_electron | is_web_frontend | has_node
+ * @returns {RuntimeCapabilities}
+ */
+function maskRuntimeForSurface(runtime, flag) {
+	/** @type {RuntimeCapabilities} */
+	const masked = {
+		is_electron: false,
+		is_tauri: false,
+		is_nextjs: false,
+		is_nuxt: false,
+		is_web_frontend: false,
+		is_expo: false,
+		is_react_native: false,
+		has_node: false,
+	}
+	// Only flip the surface's flag on if the project actually carries it
+	// (otherwise the resolver should genuinely fail — fail-closed).
+	if (flag === 'is_electron' && runtime.is_electron) masked.is_electron = true
+	if (flag === 'is_web_frontend' && runtime.is_web_frontend) {
+		masked.is_web_frontend = true
+		// Carry the framework hint too (Next.js / Nuxt) so drivers that
+		// target browser via those frameworks still match.
+		masked.is_nextjs = runtime.is_nextjs
+		masked.is_nuxt = runtime.is_nuxt
+	}
+	if (flag === 'has_node') masked.has_node = runtime.has_node
+	return masked
+}