@motionstudies/data 0.1.0-alpha.3 → 0.1.0-alpha.30

package/recorder-evidence.mjs

@@ -0,0 +1,136 @@
+import { createHash } from 'node:crypto'
+import { isAbsolute, dirname, join } from 'node:path'
+import { readOperationalFile } from './operational-files.mjs'
+import { readFeedRegistry, readFeedHealth, readOperationalTime, summarizeFeedStages } from './feed-observability.mjs'
+import { studyDay } from './uk-service-day.mjs'
+
+const hash = bytes => createHash('sha256').update(bytes).digest('hex')
+const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value)
+const count = value => Number.isSafeInteger(value) && value >= 0
+function keys(value, allowed) {
+  if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(k => !allowed.includes(k))) throw new Error('Invalid evidence plan')
+}
+function path(value) { if (typeof value !== 'string' || !isAbsolute(value) || value.length > 4096) throw new Error('Evidence paths must be absolute'); return value }
+function date(value) { readOperationalTime(`${value}T00:00:00Z`); return value }
+function readPlan(value, registry, producerId) {
+  keys(value, ['schemaVersion', 'kind', 'producerId', 'validUntil', 'feeds'])
+  if (value.schemaVersion !== 1 || value.kind !== 'recorder-evidence-plan' || value.producerId !== producerId || !Array.isArray(value.feeds) || value.feeds.length > 16) throw new Error('Invalid evidence plan')
+  const validUntil = readOperationalTime(value.validUntil)
+  const feeds = value.feeds.map(f => {
+    keys(f, ['feedId', 'archive', 'publication'])
+    const entry = registry.feeds.find(e => e.id === f.feedId && e.producerId === producerId)
+    if (!entry) throw new Error('Unregistered evidence feed')
+    const output = { feedId: f.feedId }
+    for (const id of ['archive', 'publication']) if (f[id]) {
+      if (!entry.stages.some(s => s.id === id)) throw new Error('Unregistered evidence stage')
+      if (id === 'archive') {
+        keys(f[id], ['journalPath', 'closeoutPath', 'expectedEnd', 'deadlineAt'])
+        output.archive = { journalPath: path(f[id].journalPath), closeoutPath: path(f[id].closeoutPath), expectedEnd: readOperationalTime(f[id].expectedEnd), deadlineAt: readOperationalTime(f[id].deadlineAt) }
+      } else {
+        keys(f[id], ['pointerPath', 'operator', 'expectedThrough', 'deadlineAt'])
+        if (typeof f[id].operator !== 'string' || !f[id].operator.trim() || f[id].operator.length > 100) throw new Error('Invalid operator')
+        output.publication = { pointerPath: path(f[id].pointerPath), operator: f[id].operator, expectedThrough: date(f[id].expectedThrough), deadlineAt: readOperationalTime(f[id].deadlineAt) }
+      }
+      const end = id === 'archive' ? output.archive.expectedEnd : studyDay(output.publication.expectedThrough).endUtc
+      if (Date.parse(output[id].deadlineAt) < Date.parse(end)) throw new Error('Evidence deadline precedes expected period end')
+      if (Date.parse(output[id].deadlineAt) > Date.parse(validUntil)) throw new Error('Evidence deadline exceeds plan validity')
+    }
+    return output
+  })
+  if (new Set(feeds.map(f => f.feedId)).size !== feeds.length) throw new Error('Duplicate evidence feed')
+  return { feeds, validUntil }
+}
+const result = (stageId, state, reason, timestamps = {}, metrics = {}) => ({ stageId, state, reasons: reason ? [reason] : [], timestamps, metrics })
+const pending = (id, deadline, now, reason) => result(id, now > Date.parse(deadline) ? 'degraded' : 'waiting', `${reason}-${now > Date.parse(deadline) ? 'overdue' : 'pending'}`)
+
+/** Read explicit local evidence targets; never infer paths from a status report or fetch R2. */
+export async function recorderEvidenceHealth(registryValue, reportValue, planValue) {
+  const registry = readFeedRegistry(registryValue), report = readFeedHealth(reportValue, registry)
+  const plan = readPlan(planValue, registry, report.producerId), now = Date.parse(report.observedAt)
+  // A national bus day's journal is about 140 MiB; smaller feeds are a few MiB.
+  let remaining = 512 * 1024 * 1024
+  const read = async (path, limit) => {
+    if (remaining <= 0) throw new Error('Evidence byte budget exceeded')
+    const bytes = await readOperationalFile(path, Math.min(limit, remaining))
+    remaining -= bytes.length
+    return { bytes, json: JSON.parse(bytes) }
+  }
+  for (const f of plan.feeds) {
+    const feed = report.feeds.find(e => e.feedId === f.feedId)
+    if (report.telemetry.state !== 'current' || feed.configuredState !== 'active') continue
+    const entry = registry.feeds.find(e => e.id === f.feedId)
+    for (const stageId of ['archive', 'publication']) {
+      if (!f[stageId]) continue
+      let evidence
+      try {
+        evidence = now > Date.parse(plan.validUntil) ? result(stageId, 'unknown', 'evidence-plan-expired') : stageId === 'archive' ? await archive(f.archive, entry, read, now, registry.clockSkewSeconds) : await publication(f.publication, read, now, registry.clockSkewSeconds)
+      } catch (error) {
+        evidence = error.code === 'ENOENT' ? pending(stageId, f[stageId].deadlineAt, now, `${stageId}-evidence`) : result(stageId, 'unknown', `${stageId}-evidence-invalid`)
+      }
+      feed.stages = feed.stages.map(s => s.stageId === stageId ? evidence : s)
+    }
+    feed.state = summarizeFeedStages(feed.stages)
+  }
+  return readFeedHealth(report, registry)
+}
+
+async function archive(plan, entry, read, now, skew) {
+  const { bytes, json: journal } = await read(plan.journalPath, 256 * 1024 * 1024)
+  if (!journal.id || journal.config?.source !== entry.sourceId || !['complete', 'stopped'].includes(journal.status) || !Array.isArray(journal.attempts)) throw new Error('Invalid closed journal')
+  const end = readOperationalTime(journal.plannedEnd), start = readOperationalTime(journal.startedAt)
+  if (end !== plan.expectedEnd || Date.parse(end) < Date.parse(start) || Date.parse(end) > now || Date.parse(plan.deadlineAt) < Date.parse(end)) throw new Error('Invalid archive period/deadline')
+  const dates = [...new Set(journal.attempts.filter(a => a.status === 'captured').map(a => readOperationalTime(a.receivedAt).slice(0, 10)))].sort()
+  if (!dates.length || dates.length > 7) throw new Error('Archive has no bounded captured dates')
+  const { json: record } = await read(plan.closeoutPath, 1024 * 1024)
+  if (record.journalId !== journal.id || record.source !== entry.sourceId || record.journalSha256 !== hash(bytes)) throw new Error('Closeout provenance mismatch')
+  const closedAt = readOperationalTime(record.closedAt)
+  if (Date.parse(closedAt) > now + skew * 1000 || Date.parse(closedAt) < Date.parse(start)) throw new Error('Invalid closeout clock')
+  if (JSON.stringify([...record.dates].sort()) !== JSON.stringify(dates)) throw new Error('Closeout dates differ')
+  const n = record.steps?.normalise
+  if (!n || n.unsupported || !Array.isArray(n.skipped) || n.skipped.length) return result('archive', 'degraded', 'archive-normalization-incomplete')
+  let files = 0
+  for (const d of dates) {
+    if (entry.sourceId === 'bulk-archive' && (!record.steps.packs?.[d] || record.steps.packs[d].error)) return result('archive', 'degraded', 'archive-packs-incomplete')
+    const upload = record.steps.upload?.[d]
+    if (!upload) return pending('archive', plan.deadlineAt, now, 'archive-upload')
+    for (const key of ['normalized', 'publish']) {
+      const u = upload[key]
+      if (key === 'publish' && u === null && entry.sourceId !== 'bulk-archive') continue
+      if (!u) return pending('archive', plan.deadlineAt, now, 'archive-upload')
+      if (u.error || u.stopped || u.failed > 0) return result('archive', 'degraded', 'archive-upload-failed')
+      if (![u.files, u.uploaded, u.skipped, u.failed].every(count) || u.failed !== 0 || u.files < 1 || u.uploaded + u.skipped !== u.files) throw new Error('Invalid upload accounting')
+      files += u.files
+    }
+  }
+  return result('archive', 'healthy', 'archive-closeout-verified', { latestPeriodEnd: end, lastSuccessAt: closedAt }, { 'recorded-upload-files': files })
+}
+
+async function publication(plan, read, now, skew) {
+  const { bytes: pointerBytes, json: pointer } = await read(plan.pointerPath, 8192)
+  if (pointer.schemaVersion !== 1 || pointer.kind !== 'feed-quality-release-pointer' || !digest(pointer.resultId) || pointer.operator !== plan.operator) throw new Error('Invalid release pointer')
+  const observedAt = readOperationalTime(pointer.observedAt), from = date(pointer.from), to = date(pointer.to)
+  if (Date.parse(observedAt) > now + skew * 1000 || from > to) throw new Error('Invalid publication clock/range')
+  // The caller selects the pointer; only a validated digest can select its sibling immutable artifact.
+  const dir = join(dirname(dirname(plan.pointerPath)), 'quality-releases', pointer.resultId)
+  const { json: manifest } = await read(join(dir, 'manifest.json'), 4096)
+  const file = manifest.files?.[0]
+  if (manifest.schemaVersion !== 1 || manifest.resultId !== pointer.resultId || manifest.files?.length !== 1 || file.path !== 'result.json' || !count(file.bytes) || !digest(file.sha256)) throw new Error('Invalid release manifest')
+  const { bytes, json: release } = await read(join(dir, 'result.json'), 2 * 1024 * 1024)
+  const { resultId, ...payload } = release
+  if (file.bytes !== bytes.length || file.sha256 !== hash(bytes) || resultId !== pointer.resultId || hash(JSON.stringify(payload)) !== resultId) throw new Error('Release integrity mismatch')
+  if (release.schemaVersion !== 2 || release.kind !== 'feed-quality-history' || release.operator !== plan.operator || release.from !== from || release.to !== to || !Array.isArray(release.days) || release.days.length > 366) throw new Error('Release selection mismatch')
+  if (!(await read(plan.pointerPath, 8192)).bytes.equals(pointerBytes)) throw new Error('Release selection changed during check')
+  const days = new Map()
+  for (const d of release.days) {
+    date(d.date)
+    if (days.has(d.date) || d.date < from || d.date > to) throw new Error('Invalid release day membership')
+    days.set(d.date, d)
+  }
+  if (from > plan.expectedThrough || to < plan.expectedThrough || !days.has(plan.expectedThrough)) return pending('publication', plan.deadlineAt, now, 'publication-period')
+  const selected = days.get(plan.expectedThrough)
+  const expected = studyDay(plan.expectedThrough)
+  if (Date.parse(plan.deadlineAt) < Date.parse(expected.endUtc) || Date.parse(expected.endUtc) > now + skew * 1000 || Date.parse(observedAt) < Date.parse(expected.endUtc)) throw new Error('Invalid publication deadline/period')
+  if (selected.status !== 'current') return pending('publication', plan.deadlineAt, now, 'publication-day')
+  if (!digest(selected.revision) || !selected.quality || selected.day?.serviceDate !== plan.expectedThrough || selected.day?.startUtc !== expected.startUtc || selected.day?.endUtc !== expected.endUtc) throw new Error('Invalid selected day')
+  return result('publication', 'healthy', 'publication-artifact-verified', { lastSuccessAt: observedAt, latestPeriodEnd: expected.endUtc }, { 'verified-release-bytes': bytes.length })
+}

package/recorder-observability.d.mts

@@ -0,0 +1,3 @@
+import type { FeedHealth } from './feed-observability.mjs'
+/** Reads only supplied JSON. `now` is the observer's Unix time in milliseconds. */
+export function recorderFeedHealth(registry: unknown, status: unknown, options: { producerId: string; now?: number }): FeedHealth

package/recorder-observability.mjs

@@ -0,0 +1,106 @@
+import { readFeedRegistry, readFeedHealth, readOperationalTime, summarizeFeedStages } from './feed-observability.mjs'
+
+const finite = value => Number.isFinite(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER
+const time = value => { try { return readOperationalTime(value) } catch { return null } }
+const age = (stamp, now) => (now - Date.parse(stamp)) / 1000
+const stage = (stageId, state, reasons = [], timestamps = {}, metrics = {}) => ({ stageId, state, reasons, timestamps, metrics })
+const metric = (key, value) => finite(value) ? { [key]: value } : {}
+
+/** Projection of recorder status.json. Never read paths, emit raw errors or trust cached ages/state. */
+export function recorderFeedHealth(registryValue, status, { producerId, now = Date.now() } = {}) {
+  const registry = readFeedRegistry(registryValue)
+  const observedAt = new Date(now).toISOString(), entries = registry.feeds.filter(f => f.producerId === producerId)
+  if (!entries.length) throw new Error('Unregistered producer')
+  if (entries.filter(f => f.stages.some(s => s.id === 'analytics')).length > 1) throw new Error('Recorder status supports one analytics feed')
+  const generatedAt = time(status?.generatedAt), reportAge = generatedAt ? age(generatedAt, now) : null
+  const telemetry = { state: status === null ? 'missing' : !generatedAt || reportAge < -registry.clockSkewSeconds ? 'invalid' : reportAge > registry.heartbeatMaxAgeSeconds ? 'stale' : 'current', ageSeconds: reportAge === null ? null : Math.max(0, reportAge) }
+  const unavailable = telemetry.state !== 'current'
+  if (!unavailable && (!Array.isArray(status.feeds) || status.feeds.length > 128 || new Set(status.feeds.map(f => f?.name)).size !== status.feeds.length)) throw new Error('Invalid recorder feed list')
+  const feeds = entries.map(entry => {
+    const f = status?.feeds?.find?.(f => f?.name === entry.producerFeed)
+    const stages = entry.stages.map(def => {
+      if (unavailable) return stage(def.id, 'unknown', [`telemetry-${telemetry.state}`])
+      if (entry.state !== 'active') return stage(def.id, 'unknown', [entry.state])
+      if (!f) return stage(def.id, 'unknown', ['feed-missing'])
+      if (status.storageError) return stage(def.id, 'degraded', ['host-storage-blocked'])
+      if (f.error) return stage(def.id, 'unknown', ['feed-status-unavailable'])
+      if (f.source !== entry.sourceId) return stage(def.id, 'unknown', [f.source ? 'source-mismatch' : 'source-unavailable'])
+      if (def.id === 'capture') return capture(def, f, registry, now, generatedAt)
+      if (def.id === 'normalize') return normalize(def, f, registry, now, generatedAt)
+      if (def.id === 'archive') {
+        // A complete close-out may be local-only. This report cannot prove an R2 upload.
+        const count = f.processing?.incompleteCloseouts, job = f.jobs?.[`process-${entry.producerFeed}`]
+        if (job && job.code !== 0) return stage(def.id, 'degraded', ['processing-job-failed'], {}, metric('incomplete-closeouts', count))
+        return stage(def.id, finite(count) && count > 0 ? 'waiting' : 'unknown', [finite(count) && count > 0 ? 'closeouts-pending' : 'upload-evidence-unavailable'], {}, metric('incomplete-closeouts', count))
+      }
+      if (def.id === 'analytics') return analytics(def, status.analytics, registry, now, generatedAt)
+      // Publication/serving needs separate evidence, even when a local job succeeded.
+      return stage(def.id, 'unknown', ['stage-evidence-unavailable'])
+    })
+    return { feedId: entry.id, configuredState: entry.state, state: entry.state === 'active' ? summarizeFeedStages(stages) : entry.state, stages }
+  })
+  const c = status?.capacity, disk = status?.disk
+  const capacity = unavailable ? { state: 'unknown', reasons: [`telemetry-${telemetry.state}`], metrics: {} } : {
+    state: status.storageError || c?.status === 'paused' || c?.error || c?.remainingCaptureBytes === 0 ? 'degraded' : c?.status === 'open' && finite(disk?.freeBytes) ? 'healthy' : 'unknown',
+    reasons: status.storageError ? ['host-storage-blocked'] : c?.error ? ['ledger-unavailable'] : c?.status === 'paused' ? ['ledger-paused'] : c?.remainingCaptureBytes === 0 ? ['capture-budget-exhausted'] : c?.status === 'open' && finite(disk?.freeBytes) ? [] : ['capacity-evidence-unavailable'],
+    metrics: { ...metric('disk-free-bytes', disk?.freeBytes), ...metric('disk-total-bytes', disk?.totalBytes), ...metric('committed-bytes', c?.usage?.total?.committed),
+      ...metric('reserved-bytes', c?.usage?.total?.reserved), ...metric('remaining-capture-bytes', c?.remainingCaptureBytes), ...metric('estimated-capture-days', c?.estimatedCaptureDays) },
+  }
+  return readFeedHealth({ schemaVersion: 1, kind: 'feed-health', registryId: registry.id, producerId, observedAt, producerGeneratedAt: generatedAt, telemetry, feeds, capacity }, registry)
+}
+
+function capture(def, f, registry, now, generatedAt) {
+  const receivedAt = time(f.lastCaptureAt)
+  if (!receivedAt) return stage(def.id, 'unknown', ['capture-time-unavailable'])
+  const receivedAge = age(receivedAt, now), timestamps = { receivedAt }, metrics = { 'capture-age-seconds': Math.max(0, receivedAge) }
+  if (receivedAge < -registry.clockSkewSeconds || Date.parse(receivedAt) > Date.parse(generatedAt) + registry.clockSkewSeconds * 1000) return stage(def.id, 'unknown', ['capture-clock-invalid'])
+  const reasons = []
+  if (def.maxAgeSeconds === null) return stage(def.id, 'unknown', ['capture-threshold-unconfigured'], timestamps, metrics)
+  if (receivedAge > def.maxAgeSeconds) reasons.push('capture-stale')
+  // Legacy host status stores source age at report time, not the original timestamp.
+  // Advancing it by report age prevents a fresh read from making old source data fresh.
+  if (finite(f.sourceAgeSeconds)) {
+    const sourceGeneratedAt = new Date(Date.parse(generatedAt) - f.sourceAgeSeconds * 1000)
+    if (!Number.isFinite(sourceGeneratedAt.getTime())) return stage(def.id, 'unknown', ['source-clock-invalid'], timestamps, metrics)
+    timestamps.sourceGeneratedAt = sourceGeneratedAt.toISOString()
+    metrics['source-age-seconds'] = Math.max(0, age(timestamps.sourceGeneratedAt, now))
+    if (def.sourceMaxAgeSeconds && metrics['source-age-seconds'] > def.sourceMaxAgeSeconds) reasons.push('source-stale')
+  } else if (def.sourceMaxAgeSeconds) {
+    return stage(def.id, reasons.length ? 'degraded' : 'unknown', [...reasons, 'source-time-unavailable'], timestamps, metrics)
+  }
+  if (f.plan?.action === 'blocked') reasons.push('collection-blocked')
+  if (reasons.length) return stage(def.id, 'degraded', reasons, timestamps, metrics)
+  if (['complete', 'stopped'].includes(f.state) || f.plan?.action === 'wait') return stage(def.id, 'waiting', ['collection-between-runs'], timestamps, metrics)
+  if (!['healthy', 'stalled'].includes(f.state)) return stage(def.id, 'unknown', ['collection-state-unrecognized'], timestamps, metrics)
+  return stage(def.id, 'healthy', [], timestamps, metrics)
+}
+
+function normalize(def, f, registry, now, generatedAt) {
+  const hour = f.processing?.latestNormalizedHour
+  const start = typeof hour === 'string' && /^\d{4}-\d\d-\d\dT\d\d$/.test(hour) ? time(`${hour}:00:00Z`) : null
+  if (!start) return stage(def.id, 'unknown', ['normalized-period-unavailable'])
+  const latestPeriodEnd = new Date(Date.parse(start) + 3600000).toISOString(), lag = age(latestPeriodEnd, now)
+  if (lag < -registry.clockSkewSeconds || Date.parse(latestPeriodEnd) > Date.parse(generatedAt) + registry.clockSkewSeconds * 1000) return stage(def.id, 'unknown', ['normalized-period-in-future'])
+  const timestamps = { latestPeriodEnd }, metrics = { 'normalization-lag-seconds': Math.max(0, lag) }
+  if (def.maxAgeSeconds === null) return stage(def.id, 'unknown', ['normalization-threshold-unconfigured'], timestamps, metrics)
+  return stage(def.id, lag > def.maxAgeSeconds ? 'degraded' : 'healthy', lag > def.maxAgeSeconds ? ['normalization-lagging'] : [], timestamps, metrics)
+}
+
+function analytics(def, a, registry, now, generatedAt) {
+  const attemptedAt = time(a?.observedAt)
+  if (!attemptedAt) return stage(def.id, 'unknown', ['analytics-status-unavailable'])
+  const elapsed = age(attemptedAt, now)
+  if (elapsed < -registry.clockSkewSeconds || Date.parse(attemptedAt) > Date.parse(generatedAt) + registry.clockSkewSeconds * 1000) return stage(def.id, 'unknown', ['analytics-clock-invalid'])
+  const timestamps = { lastAttemptAt: attemptedAt }
+  if (def.maxAgeSeconds === null) return stage(def.id, 'unknown', ['analytics-threshold-unconfigured'], timestamps)
+  if (elapsed > def.maxAgeSeconds) return stage(def.id, 'unknown', ['analytics-status-stale'], timestamps)
+  if (a.error || a.job && a.job.code !== 0) return stage(def.id, 'degraded', ['analytics-job-failed'], timestamps)
+  if (!Array.isArray(a.pendingMonths) || !Array.isArray(a.waiting) || !a.months || typeof a.months !== 'object' || Array.isArray(a.months) || Object.keys(a.months).length > 120) return stage(def.id, 'unknown', ['analytics-status-invalid'], timestamps)
+  const months = Object.values(a.months), successes = months.map(m => time(m?.lastSuccessAt)).filter(Boolean).sort()
+  if (successes.length) timestamps.lastSuccessAt = successes.at(-1)
+  if (timestamps.lastSuccessAt && (age(timestamps.lastSuccessAt, now) < -registry.clockSkewSeconds || Date.parse(timestamps.lastSuccessAt) > Date.parse(generatedAt) + registry.clockSkewSeconds * 1000)) return stage(def.id, 'unknown', ['analytics-clock-invalid'])
+  const metrics = { 'pending-months': a.pendingMonths.length, 'waiting-inputs': a.waiting.length }
+  // Open receipt hours normally remain waiting. A fresh worker check with no work is not a failed publication.
+  const waiting = a.pendingMonths.length > 0 || a.waiting.length > 0
+  return stage(def.id, waiting ? 'waiting' : successes.length ? 'healthy' : 'unknown', waiting ? ['analytics-awaiting-inputs-or-work'] : successes.length ? [] : ['analytics-success-unavailable'], timestamps, metrics)
+}

package/release-files.d.mts

@@ -0,0 +1,14 @@
+import type { ReleaseDescriptor } from './release.mjs'
+export interface ReleaseOpenOptions { kind?: string; schemaVersions?: readonly number[]; manifestSha256?: string; maxManifestBytes?: number; manifest?: string }
+export interface OpenedRelease<M = Record<string, unknown>> {
+  manifest: M
+  manifestBytes: Buffer
+  descriptors: ReleaseDescriptor[]
+  /** Read a file the manifest describes, by path or descriptor, checking size and digest. */
+  read(file: string | ReleaseDescriptor, options?: { maxBytes?: number }): Promise<Buffer>
+}
+/** Bounded read without following a final symlink or blocking on a FIFO. */
+export function readOperationalFile(path: string, limit: number): Promise<Buffer>
+export function readReleaseFile(root: string, descriptor: ReleaseDescriptor, options?: { maxBytes?: number }): Promise<Buffer>
+export function openRelease<M = Record<string, unknown>>(root: string, options?: ReleaseOpenOptions): Promise<OpenedRelease<M>>
+export function verifyRelease<M = Record<string, unknown>>(root: string, options?: ReleaseOpenOptions & { maxTotalBytes?: number }): Promise<{ manifest: M; manifestSha256: string; files: number; bytes: number }>

package/release-files.mjs

@@ -0,0 +1,52 @@
+import { realpath } from 'node:fs/promises'
+import { join, relative, isAbsolute, resolve, sep } from 'node:path'
+import { readOperationalFile } from './operational-files.mjs'
+import { checkReleaseManifest, digestHex, isReleasePath, readDescriptor, releaseDescriptors, verifyDescriptorBytes } from './release.mjs'
+
+export { readOperationalFile }
+const inside = (root, path) => { const r = relative(root, path); return r !== '' && !r.startsWith(`..${sep}`) && r !== '..' && !isAbsolute(r) }
+
+/**
+ * Read one described file from a release directory: safe path, bounded, size and digest checked. No component below
+ * the release root may be a symbolic link, even one pointing elsewhere inside it, so a described path always names
+ * exactly the file that was written there.
+ */
+export async function readReleaseFile(root, descriptor, { maxBytes = 512 * 1024 ** 2 } = {}) {
+  const d = readDescriptor(descriptor)
+  if (d.bytes > maxBytes) throw new Error(`${d.path} exceeds the read limit`)
+  const base = await realpath(resolve(root)), target = join(base, ...d.path.split('/')), actual = await realpath(target)
+  if (!inside(base, actual)) throw new Error(`${d.path} resolves outside the release`)
+  if (actual !== target) throw new Error(`${d.path} passes through a symbolic link`)
+  return verifyDescriptorBytes(await readOperationalFile(actual, d.bytes), d)
+}
+
+/**
+ * Open a release directory: read `manifest.json` (bounded), optionally require its pinned digest, check its kind and
+ * schema, and enumerate its files. Nothing but the manifest is read until asked.
+ */
+export async function openRelease(root, { kind, schemaVersions, manifestSha256, maxManifestBytes = 8 * 1024 ** 2, manifest: name = 'manifest.json' } = {}) {
+  if (!isReleasePath(name)) throw new Error('Unsafe manifest path')
+  const bytes = await readOperationalFile(join(resolve(root), name), maxManifestBytes)
+  if (manifestSha256 !== undefined && await digestHex(bytes) !== manifestSha256) throw new Error('Release manifest does not match its pinned digest')
+  const manifest = checkReleaseManifest(JSON.parse(bytes.toString('utf8')), { kind, schemaVersions })
+  const descriptors = releaseDescriptors(manifest), byPath = new Map(descriptors.map(d => [d.path, d]))
+  return {
+    manifest, manifestBytes: bytes, descriptors,
+    /** Read a described file by path or descriptor; a path the manifest does not list is refused. */
+    read: async (file, options) => {
+      const d = typeof file === 'string' ? byPath.get(file) : file
+      const listed = d && byPath.get(d.path)
+      if (!listed || listed.sha256 !== d.sha256 || listed.bytes !== d.bytes) throw new Error(`${typeof file === 'string' ? file : file?.path} is not described by the manifest`)
+      return readReleaseFile(root, d, options)
+    },
+  }
+}
+
+/** Verify every file a release describes against a total byte budget. */
+export async function verifyRelease(root, { maxTotalBytes = 2 * 1024 ** 3, ...options } = {}) {
+  const release = await openRelease(root, options)
+  const total = release.descriptors.reduce((n, d) => n + d.bytes, 0)
+  if (total > maxTotalBytes) throw new Error('Release exceeds the verification budget')
+  for (const d of release.descriptors) await readReleaseFile(root, d)
+  return { manifest: release.manifest, manifestSha256: await digestHex(release.manifestBytes), files: release.descriptors.length, bytes: total }
+}

package/release.d.mts

@@ -0,0 +1,10 @@
+/** A file a release describes: its release-relative path and the size and SHA-256 of its stored bytes. */
+export interface ReleaseDescriptor { path: string; sha256: string; bytes: number; [field: string]: unknown }
+export declare const RELEASE_DIGEST: RegExp
+export function isReleasePath(path: unknown): path is string
+export function readDescriptor(value: unknown, label?: string): ReleaseDescriptor
+/** Every descriptor in a manifest's `files` (array or map), `objects`, `chunks`, `list` and named top-level descriptors, sorted by path. */
+export function releaseDescriptors(manifest: unknown): ReleaseDescriptor[]
+export function checkReleaseManifest<T extends object>(manifest: T, options: { kind?: string; schemaVersions?: readonly number[] }): T
+export function digestHex(bytes: ArrayBuffer | ArrayBufferView): Promise<string>
+export function verifyDescriptorBytes<T extends ArrayBuffer | ArrayBufferView>(bytes: T, descriptor: ReleaseDescriptor): Promise<T>

package/release.mjs

@@ -0,0 +1,62 @@
+// Shared checks for versioned release handoffs. Runs in Node and in browsers (Web Crypto only).
+
+export const RELEASE_DIGEST = /^[a-f0-9]{64}$/
+const SEGMENT = /^[A-Za-z0-9._-]{1,255}$/
+
+/** A release-relative path: forward-slash segments of safe characters, never absolute, empty, `.` or `..`. */
+export function isReleasePath(path) {
+  return typeof path === 'string' && path.length <= 1024 && path.split('/').every(s => SEGMENT.test(s) && s !== '.' && s !== '..')
+}
+
+/** Validate one `{ path, sha256, bytes }` file descriptor; extra fields such as `decodedBytes` are kept. */
+export function readDescriptor(value, label = 'file') {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid ${label} descriptor`)
+  if (!isReleasePath(value.path)) throw new Error(`Unsafe ${label} path`)
+  if (!RELEASE_DIGEST.test(value.sha256 ?? '')) throw new Error(`Invalid ${label} digest`)
+  if (!Number.isSafeInteger(value.bytes) || value.bytes < 0) throw new Error(`Invalid ${label} size`)
+  return value
+}
+
+const looksLikeDescriptor = v => v && typeof v === 'object' && !Array.isArray(v) && typeof v.path === 'string' && 'sha256' in v && 'bytes' in v
+/**
+ * Every file a manifest describes, whatever its layout: a `files` array or name-keyed map, an `objects` map keyed by
+ * digest, `chunks`, a published bus day's `list`, and named top-level descriptors such as `day` or `index`. The same
+ * path may appear more than once (air releases list chunks in `files` and `chunks`) but must then describe the same
+ * bytes.
+ */
+export function releaseDescriptors(manifest) {
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) throw new Error('Invalid release manifest')
+  const found = new Map()
+  const add = (value, label) => {
+    const d = readDescriptor(value, label), seen = found.get(d.path)
+    if (seen && (seen.sha256 !== d.sha256 || seen.bytes !== d.bytes)) throw new Error(`Conflicting descriptors for ${d.path}`)
+    if (!seen) found.set(d.path, d)
+  }
+  for (const [key, value] of Object.entries(manifest)) {
+    if (Array.isArray(value) && ['files', 'chunks', 'list'].includes(key)) value.forEach((v, i) => add(v, `${key}[${i}]`))
+    else if (['files', 'objects'].includes(key) && value && typeof value === 'object' && !Array.isArray(value)) for (const [name, v] of Object.entries(value)) add(v, `${key}.${name}`)
+    else if (looksLikeDescriptor(value)) add(value, key)
+  }
+  return [...found.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
+}
+
+/** Check a manifest's identity before trusting its layout. */
+export function checkReleaseManifest(manifest, { kind, schemaVersions }) {
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) throw new Error('Invalid release manifest')
+  if (kind !== undefined && manifest.kind !== kind) throw new Error(`Expected a ${kind} release`)
+  if (schemaVersions !== undefined && !schemaVersions.includes(manifest.schemaVersion)) throw new Error(`Unsupported ${manifest.kind ?? 'release'} schema version`)
+  return manifest
+}
+
+const hex = buffer => Array.from(new Uint8Array(buffer), b => b.toString(16).padStart(2, '0')).join('')
+/** SHA-256 of bytes as lowercase hex, through Web Crypto. */
+export async function digestHex(bytes) {
+  return hex(await globalThis.crypto.subtle.digest('SHA-256', bytes))
+}
+/** Throw unless `bytes` are exactly the descriptor's size and digest. */
+export async function verifyDescriptorBytes(bytes, descriptor) {
+  const d = readDescriptor(descriptor)
+  if (bytes.byteLength !== d.bytes) throw new Error(`${d.path} has ${bytes.byteLength} bytes, expected ${d.bytes}`)
+  if (await digestHex(bytes) !== d.sha256) throw new Error(`${d.path} does not match its digest`)
+  return bytes
+}

package/source-store.d.mts

@@ -0,0 +1,15 @@
+import type {Buffer} from 'node:buffer'
+export interface CaptureSource {id:string;publisher:string;kind:string;documentation:string;license:string;attribution?:string;origins:readonly string[]}
+export interface CaptureRecord {schemaVersion:1;source:Omit<CaptureSource,'origins'>;url:string;sha256:string;bytes:number;storedBytes?:number;encoding?:'gzip'|null;retrievedAt:string|null;importedAt?:string;acquisition:'http'|'local-import';validation:string;httpStatus?:number;contentType?:string|null;lastModified?:string|null;etag?:string|null}
+export interface CaptureResult {bytes:Buffer;record:CaptureRecord;cached:boolean}
+export type CaptureValidator=(bytes:Buffer)=>unknown|Promise<unknown>
+export interface CaptureOptions {store:string;source:CaptureSource;url:string;offline?:boolean;refresh?:boolean;maxBytes?:number;timeoutMs?:number;retries?:number;validate?:CaptureValidator;fetchImpl?:typeof fetch;userAgent?:string;compression?:'gzip'}
+export function sha256(bytes:string|Uint8Array):string
+export function publicSourceUrl(source:CaptureSource,value:string):string
+export function atomicJson(path:string,value:unknown):Promise<void>
+export function readCapture(store:string,source:CaptureSource,url:string,validate?:CaptureValidator):Promise<CaptureResult>
+export function importCapture(options:Pick<CaptureOptions,'store'|'source'|'url'|'maxBytes'|'validate'|'compression'>&{input:string}):Promise<CaptureResult>
+export function readObject(store:string,hash:string,expectedBytes?:number):Promise<Buffer>
+export function compactObjects(store:string,options?:{minimumSaving?:number}):Promise<{examined:number;compressed:number;kept:number;rawBytes:number;storedBytes:number}>
+export function capture(options:CaptureOptions):Promise<CaptureResult>
+export function jsonDocument(bytes:Buffer):Record<string,unknown>|unknown[]

package/source-store.mjs

@@ -0,0 +1,160 @@
+import { createHash, randomUUID } from 'node:crypto'
+import { mkdir, readFile, writeFile, rename, rm, stat, readdir, unlink } from 'node:fs/promises'
+import { join, dirname } from 'node:path'
+import { setTimeout as delay } from 'node:timers/promises'
+import { gzipSync, gunzipSync } from 'node:zlib'
+
+export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex')
+
+export function publicSourceUrl(source, value) {
+  const url = new URL(value)
+  if (url.protocol !== 'https:' || url.username || url.password || url.hash ||
+      [...url.searchParams.keys()].some(key => /key|token|password|secret|signature|credential/i.test(key))) {
+    throw new Error('Source URLs must be public HTTPS URLs without credentials or fragments')
+  }
+  if (!source.origins.includes(url.origin)) throw new Error(`URL origin is not configured for ${source.id}`)
+  return url.href
+}
+
+export async function atomicJson(path, value) {
+  await atomicWrite(path, Buffer.from(`${JSON.stringify(value, null, 2)}\n`))
+}
+
+async function atomicWrite(path, bytes) {
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 })
+  const temporary = `${path}.${randomUUID()}.partial`
+  try {
+    await writeFile(temporary, bytes, { mode: 0o600 })
+    await rename(temporary, path)
+  } finally {
+    await rm(temporary, { force: true })
+  }
+}
+
+function requestKey(source, url) { return sha256(`${source.id}\n${url}`) }
+
+export async function readCapture(store, source, url, validate = () => {}) {
+  const record = JSON.parse(await readFile(join(store, 'requests', `${requestKey(source, url)}.json`), 'utf8'))
+  if (record.source.id !== source.id || record.url !== url || !/^[a-f0-9]{64}$/.test(record.sha256)) throw new Error('Capture identity mismatch')
+  const bytes = await readObject(store, record.sha256, record.bytes)
+  if (bytes.length !== record.bytes || sha256(bytes) !== record.sha256) throw new Error('Capture integrity check failed')
+  await validate(bytes)
+  return { bytes, record, cached: true }
+}
+/** Read an object by its raw-content hash, whether it is stored raw or gzip-compressed at rest. */
+export async function readObject(store, hash, expectedBytes) {
+  try { return await readFile(join(store, 'objects', hash)) } catch (error) { if (error.code !== 'ENOENT') throw error }
+  const compressed = await readFile(join(store, 'objects', `${hash}.gz`))
+  try { return gunzipSync(compressed, { maxOutputLength: (expectedBytes ?? 512 * 1024 * 1024) + 1 }) } catch { throw new Error('Compressed capture could not be expanded within its recorded size') }
+}
+/** Store object bytes, compressing at rest when asked and worthwhile; the hash of the raw bytes remains the identity. */
+async function writeObject(store, hash, bytes, compression) {
+  if (compression === 'gzip') {
+    const compressed = gzipSync(bytes, { level: 6 })
+    if (compressed.length <= bytes.length * 0.9) { await atomicWrite(join(store, 'objects', `${hash}.gz`), compressed); return { storedBytes: compressed.length, encoding: 'gzip' } }
+  } else if (compression !== undefined) throw new Error('Unsupported compression; use gzip or leave unset')
+  await atomicWrite(join(store, 'objects', hash), bytes)
+  return { storedBytes: bytes.length, encoding: null }
+}
+/** Compress existing raw objects in a store where gzip saves at least the given fraction. Records keep their raw hash and byte count; reads follow either form. */
+export async function compactObjects(store, { minimumSaving = 0.1 } = {}) {
+  const result = { examined: 0, compressed: 0, kept: 0, rawBytes: 0, storedBytes: 0 }
+  let names
+  try { names = await readdir(join(store, 'objects')) } catch (error) { if (error.code === 'ENOENT') return result; throw error }
+  for (const name of names.filter(n => /^[a-f0-9]{64}$/.test(n))) {
+    const bytes = await readFile(join(store, 'objects', name))
+    result.examined++; result.rawBytes += bytes.length
+    const compressed = gzipSync(bytes, { level: 6 })
+    if (compressed.length <= bytes.length * (1 - minimumSaving)) {
+      await atomicWrite(join(store, 'objects', `${name}.gz`), compressed)
+      if (sha256(gunzipSync(compressed)) !== name) throw new Error('Compressed object failed verification; raw object retained')
+      await unlink(join(store, 'objects', name)); result.compressed++; result.storedBytes += compressed.length
+    } else { result.kept++; result.storedBytes += bytes.length }
+  }
+  return result
+}
+
+async function persist(store, source, url, bytes, acquisition, response = {}, compression) {
+  const hash = sha256(bytes)
+  const stored = await writeObject(store, hash, bytes, compression)
+  const record = {
+    schemaVersion: 1, source: { id: source.id, publisher: source.publisher, kind: source.kind, documentation: source.documentation, license: source.license, ...(source.attribution ? { attribution: source.attribution } : {}) },
+    url, sha256: hash, bytes: bytes.length, storedBytes: stored.storedBytes, encoding: stored.encoding,
+    ...(acquisition === 'http' ? { retrievedAt: new Date().toISOString() } : { retrievedAt: null, importedAt: new Date().toISOString() }),
+    acquisition, validation: 'raw-capture; see adapter audit for fitness', ...response,
+  }
+  await atomicJson(join(store, 'records', `${randomUUID()}.json`), record)
+  await atomicJson(join(store, 'requests', `${requestKey(source, url)}.json`), record)
+  return { bytes, record, cached: false }
+}
+
+export async function importCapture({ store, source, url, input, maxBytes = 512 * 1024 * 1024, validate = () => {}, compression }) {
+  url = publicSourceUrl(source, url)
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new Error('Expected a positive byte limit')
+  if ((await stat(input)).size > maxBytes) throw new Error('Source file exceeds byte limit')
+  const bytes = await readFile(input)
+  if (!bytes.length || bytes.length > maxBytes) throw new Error('Source file is empty or exceeds byte limit')
+  await validate(bytes)
+  return persist(store, source, url, bytes, 'local-import', {}, compression)
+}
+
+export async function capture({ store, source, url, offline = false, refresh = false, maxBytes = 32 * 1024 * 1024, timeoutMs = 20000, retries = 2, validate = () => {}, fetchImpl = fetch, userAgent = 'Motion Studies source capture', compression }) {
+  url = publicSourceUrl(source, url)
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || !Number.isSafeInteger(retries) || retries < 0 || retries > 10) throw new Error('Invalid capture limits')
+  if (offline && refresh) throw new Error('Offline and refresh cannot be combined')
+  if (!refresh) {
+    try { return await readCapture(store, source, url, validate) }
+    catch (error) { if (error.code !== 'ENOENT') throw error }
+  }
+  if (offline) throw new Error(`No cached capture for ${source.id}; offline mode never downloads`)
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    let retryable = true
+    try {
+      const response = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs), headers: { 'User-Agent': userAgent } })
+      if (response.status === 204) {
+        retryable = false
+        throw new Error('Source returned HTTP 204: no report content; this does not mean zero traffic')
+      }
+      if (!response.ok) {
+        retryable = response.status === 429 || response.status >= 500
+        await response.body?.cancel()
+        throw new Error(`Source returned HTTP ${response.status}`)
+      }
+      if (Number(response.headers.get('content-length')) > maxBytes) {
+        retryable = false
+        await response.body?.cancel()
+        throw new Error('Source exceeds byte limit')
+      }
+      if (!response.body) throw new Error('Source response has no body')
+      const reader = response.body.getReader()
+      const chunks = []
+      let size = 0
+      try {
+        while (true) {
+          const { value, done } = await reader.read()
+          if (done) break
+          size += value.length
+          if (size > maxBytes) { retryable = false; await reader.cancel(); throw new Error('Source exceeds byte limit') }
+          chunks.push(value)
+        }
+      } finally { reader.releaseLock() }
+      const bytes = Buffer.concat(chunks)
+      retryable = false
+      if (!bytes.length) throw new Error('Source response is empty')
+      await validate(bytes)
+      return await persist(store, source, url, bytes, 'http', {
+        httpStatus: response.status, contentType: response.headers.get('content-type'),
+        lastModified: response.headers.get('last-modified'), etag: response.headers.get('etag'),
+      }, compression)
+    } catch (error) {
+      if (!retryable || attempt === retries) throw error
+      await delay(250 * 2 ** attempt)
+    }
+  }
+}
+
+export function jsonDocument(bytes) {
+  const value = JSON.parse(bytes.toString('utf8'))
+  if (!value || typeof value !== 'object') throw new Error('Expected a JSON document')
+  return value
+}

package/uk-service-day.d.mts

@@ -0,0 +1,7 @@
+export declare const SERVICE_TIMEZONE: 'Europe/London'
+/** UK civil day boundaries. GTFS service time has a separate noon-based origin. */
+export interface UkStudyDay {serviceDate:string;timezone:'Europe/London';startUtc:string;endUtc:string;durationSeconds:number}
+export function checkedDate(value:string):string
+export function studyDay(serviceDate:string):UkStudyDay
+export function utcDatesForDay(day:UkStudyDay):string[]
+export function gtfsServiceInstant(serviceDate:string,seconds:number):string

package/uk-service-day.mjs

@@ -0,0 +1,49 @@
+export const SERVICE_TIMEZONE = 'Europe/London'
+
+export function checkedDate(value) {
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(value ?? '') || !Number.isFinite(Date.parse(`${value}T00:00:00Z`)) ||
+      new Date(`${value}T00:00:00Z`).toISOString().slice(0, 10) !== value) {
+    throw new Error('An actual service date in YYYY-MM-DD form is required')
+  }
+  return value
+}
+
+// UK midnight is unambiguous on both DST transition days. Never infer a full day as 86400 seconds.
+function londonMidnight(date) {
+  const nominal = Date.parse(`${date}T00:00:00Z`)
+  const formatter = new Intl.DateTimeFormat('en-GB', { timeZone: SERVICE_TIMEZONE, timeZoneName: 'longOffset' })
+  let instant = nominal
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const offset = formatter.formatToParts(instant).find(part => part.type === 'timeZoneName').value
+    const match = /^GMT([+-])(\d{2}):(\d{2})$/.exec(offset)
+    if (offset !== 'GMT' && !match) throw new Error('Unsupported timezone offset')
+    const minutes = match ? (match[1] === '+' ? 1 : -1) * (Number(match[2]) * 60 + Number(match[3])) : 0
+    instant = nominal - minutes * 60000
+  }
+  return instant
+}
+
+export function studyDay(serviceDate) {
+  checkedDate(serviceDate)
+  const next = new Date(Date.parse(`${serviceDate}T00:00:00Z`) + 86400000).toISOString().slice(0, 10)
+  const start = londonMidnight(serviceDate)
+  const end = londonMidnight(next)
+  return { serviceDate, timezone: SERVICE_TIMEZONE, startUtc: new Date(start).toISOString(), endUtc: new Date(end).toISOString(), durationSeconds: (end - start) / 1000 }
+}
+
+export function utcDatesForDay(day) {
+  const first = Date.parse(day.startUtc.slice(0, 10))
+  const last = Date.parse(new Date(Date.parse(day.endUtc) - 1).toISOString().slice(0, 10))
+  const dates = []
+  for (let time = first; time <= last; time += 86400000) dates.push(new Date(time).toISOString().slice(0, 10))
+  return dates
+}
+
+// GTFS defines service time from local noon minus twelve elapsed hours. On UK
+// DST transition days this differs from the observed civil-day midnight.
+export function gtfsServiceInstant(serviceDate, seconds) {
+  if (!Number.isSafeInteger(seconds) || seconds < 0) throw new Error('GTFS seconds must be a nonnegative whole number')
+  const day = studyDay(serviceDate)
+  const noonMinusTwelveHours = Date.parse(day.endUtc) - 86400000
+  return new Date(noonMinusTwelveHours + seconds * 1000).toISOString()
+}