@motionstudies/data 0.1.0-alpha.3 → 0.1.0-alpha.30

package/feed-observer.mjs

@@ -0,0 +1,192 @@
+import { readFeedRegistry, readFeedHealth, readOperationalTime, summarizeFeedStages } from './feed-observability.mjs'
+import { recorderFeedHealth } from './recorder-observability.mjs'
+import { dailyFeedExpectation } from './feed-expectations.mjs'
+import { studyDay, SERVICE_TIMEZONE } from './uk-service-day.mjs'
+
+export const OBSERVER_LIMITS = Object.freeze({ requests: 9, totalBytes: 10 * 1024 * 1024, timeoutMs: 5000, totalMs: 30000, healthBytes: 1024 * 1024, manifestBytes: 8192, releaseBytes: 2 * 1024 * 1024 })
+const digest = s => typeof s === 'string' && /^[a-f0-9]{64}$/.test(s)
+const sha = async bytes => [...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))].map(b => b.toString(16).padStart(2, '0')).join('')
+const stage = (state, reason, timestamps = {}, metrics = {}) => ({ stageId: 'serve', state, reasons: [reason], timestamps, metrics })
+function checkUrl(value) {
+  const url = new URL(value)
+  if (url.username || url.password || url.hash || url.search || !['https:', 'http:'].includes(url.protocol) || url.protocol === 'http:' && !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) throw new Error('Use HTTPS or loopback HTTP without embedded credentials or query strings')
+  return url.href
+}
+function config(value, registry) {
+  if (value?.schemaVersion !== 1 || value.kind !== 'feed-observer-config' || Object.keys(value).some(k => !['schemaVersion', 'kind', 'producerId', 'healthUrl', 'consumers'].includes(k))) throw new Error('Invalid observer config')
+  if (!registry.feeds.some(f => f.producerId === value.producerId) || !Array.isArray(value.consumers) || value.consumers.length > 4) throw new Error('Invalid observer targets')
+  const consumers = value.consumers.map(c => {
+    if (!c || Object.keys(c).some(k => !['feedId', 'manifestUrl', 'operator', 'deadline', 'timeZone'].includes(k))) throw new Error('Invalid consumer config')
+    if (!registry.feeds.some(f => f.id === c.feedId && f.producerId === value.producerId && f.stages.some(s => s.id === 'serve'))) throw new Error('Register the consumer serve stage')
+    if (typeof c.operator !== 'string' || !c.operator.trim() || c.operator.length > 100 || c.timeZone !== SERVICE_TIMEZONE || !/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(c.deadline)) throw new Error('Invalid bus consumer expectation')
+    return { ...c, manifestUrl: checkUrl(c.manifestUrl) }
+  })
+  if (new Set(consumers.map(c => c.feedId)).size !== consumers.length) throw new Error('Duplicate consumer target')
+  return { producerId: value.producerId, healthUrl: checkUrl(value.healthUrl), consumers }
+}
+class CheckError extends Error { constructor(reason) { super(reason); this.reason = reason } }
+function client(fetchImpl, tokens, limits) {
+  const began = performance.now(), usage = { requests: 0, bytes: 0 }
+  const read = async (url, cap, token) => {
+    if (usage.requests >= limits.requests || usage.bytes >= limits.totalBytes || performance.now() - began >= limits.totalMs) throw new CheckError('check-budget-exhausted')
+    usage.requests++
+    const controller = new AbortController()
+    let reader, body, timer
+    const timeout = new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(new CheckError('request-timeout')) }, Math.min(limits.timeoutMs, limits.totalMs - (performance.now() - began))) })
+    try {
+      return await Promise.race([timeout, (async () => {
+        const credential = typeof token === 'string' ? { bearer: token } : token ?? {}
+        const headers = { Accept: 'application/json', 'Cache-Control': 'no-cache', ...(credential.bearer ? { Authorization: `Bearer ${credential.bearer}` } : {}) }
+        if (credential.accessClientId && credential.accessClientSecret) {
+          headers['CF-Access-Client-Id'] = credential.accessClientId
+          headers['CF-Access-Client-Secret'] = credential.accessClientSecret
+        } else if (credential.accessClientId || credential.accessClientSecret) throw new CheckError('credentials-invalid')
+        const response = await fetchImpl(url, { headers, signal: controller.signal, redirect: 'error', cache: 'no-store' })
+        body = response.body
+        if (!response.ok || response.redirected) throw new CheckError('http-unavailable')
+        const advertised = response.headers.get('content-length')
+        if (advertised !== null && (!/^\d+$/.test(advertised) || Number(advertised) > cap)) throw new CheckError('response-too-large')
+        reader = response.body?.getReader()
+        if (!reader) throw new CheckError('document-invalid')
+        let size = 0
+        const chunks = []
+        while (true) {
+          const { done, value } = await reader.read()
+          if (done) break
+          size += value.byteLength; usage.bytes += value.byteLength
+          if (size > cap || usage.bytes > limits.totalBytes) throw new CheckError('response-too-large')
+          chunks.push(value)
+        }
+        const bytes = new Uint8Array(size)
+        let offset = 0
+        for (const c of chunks) { bytes.set(c, offset); offset += c.length }
+        let json
+        try { json = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) } catch { throw new CheckError('document-invalid') }
+        return { bytes, json }
+      })()])
+    } catch (e) { throw e instanceof CheckError ? e : new CheckError('request-failed') }
+    finally { clearTimeout(timer); controller.abort(); if (reader) void reader.cancel().catch(() => {}); else if (body) void body.cancel().catch(() => {}) }
+  }
+  return { usage, health: url => read(url, limits.healthBytes, tokens.health), consumer: (url, kind, token) => read(url, limits[`${kind}Bytes`], token) }
+}
+
+/** Reevaluate a mirrored producer report at the checker's clock; never renew its heartbeat. */
+function refresh(registry, raw, producerId, now) {
+  const report = readFeedHealth(raw, registry)
+  if (report.producerId !== producerId || Date.parse(report.observedAt) > now + registry.clockSkewSeconds * 1000) throw new Error('Unexpected producer/observer clock')
+  const generated = report.producerGeneratedAt ? Date.parse(report.producerGeneratedAt) : null
+  const elapsed = generated === null ? null : (now - generated) / 1000
+  report.observedAt = new Date(now).toISOString()
+  report.telemetry = { state: elapsed === null ? report.telemetry.state : elapsed < -registry.clockSkewSeconds ? 'invalid' : elapsed > registry.heartbeatMaxAgeSeconds ? 'stale' : 'current', ageSeconds: elapsed === null ? null : Math.max(0, elapsed) }
+  for (const f of report.feeds) {
+    const entry = registry.feeds.find(e => e.id === f.feedId)
+    f.stages = f.stages.map(s => {
+      if (report.telemetry.state !== 'current') return { ...s, state: 'unknown', reasons: [`telemetry-${report.telemetry.state}`], timestamps: {}, metrics: {} }
+      const def = entry.stages.find(d => d.id === s.stageId)
+      const field = { capture: 'receivedAt', normalize: 'latestPeriodEnd', analytics: 'lastAttemptAt' }[s.stageId]
+      const metric = { capture: 'capture-age-seconds', normalize: 'normalization-lag-seconds', analytics: 'analytics-status-age-seconds' }[s.stageId]
+      if (field && s.timestamps[field]) s.metrics[metric] = Math.max(0, (now - Date.parse(s.timestamps[field])) / 1000)
+      if (s.stageId === 'capture' && s.timestamps.sourceGeneratedAt) s.metrics['source-age-seconds'] = Math.max(0, (now - Date.parse(s.timestamps.sourceGeneratedAt)) / 1000)
+      if (f.configuredState !== 'active' || !['healthy', 'waiting'].includes(s.state)) return s
+      if (field && def.maxAgeSeconds !== null) {
+        const stamp = s.timestamps[field], age = stamp ? (now - Date.parse(stamp)) / 1000 : null
+        if (age === null) return { ...s, state: 'unknown', reasons: ['stage-clock-unavailable'] }
+        if (age < -registry.clockSkewSeconds) return { ...s, state: 'unknown', reasons: ['stage-clock-invalid'] }
+        if (age > def.maxAgeSeconds) return { ...s, state: s.stageId === 'analytics' ? 'unknown' : 'degraded', reasons: [`${s.stageId}-stale`] }
+      }
+      if (s.stageId === 'capture' && def.sourceMaxAgeSeconds) {
+        const stamp = s.timestamps.sourceGeneratedAt
+        if (!stamp) return { ...s, state: 'unknown', reasons: ['source-time-unavailable'] }
+        if ((now - Date.parse(stamp)) / 1000 < -registry.clockSkewSeconds) return { ...s, state: 'unknown', reasons: ['source-clock-invalid'] }
+        if ((now - Date.parse(stamp)) / 1000 > def.sourceMaxAgeSeconds) return { ...s, state: 'degraded', reasons: ['source-stale'] }
+      }
+      return s
+    })
+    f.state = f.configuredState === 'active' ? summarizeFeedStages(f.stages) : f.configuredState
+  }
+  if (report.telemetry.state !== 'current') report.capacity = { state: 'unknown', reasons: [`telemetry-${report.telemetry.state}`], metrics: {} }
+  return readFeedHealth(report, registry)
+}
+
+async function consumer(c, expectation, api, now, skew, token) {
+  try {
+    const { json: manifest } = await api.consumer(c.manifestUrl, 'manifest', token)
+    const id = manifest.resultId, file = manifest.report
+    if (manifest.schemaVersion !== 1 || !digest(id) || file?.path !== `releases/${id}/result.json` || !Number.isSafeInteger(file.bytes) || file.bytes < 1 || file.bytes > OBSERVER_LIMITS.releaseBytes || !digest(file.sha256)) throw new CheckError('manifest-invalid')
+    const publishedAt = readOperationalTime(manifest.observedAt)
+    if (Date.parse(publishedAt) > now + skew * 1000) throw new CheckError('publication-clock-invalid')
+    const { json: release, bytes } = await api.consumer(new URL(file.path, c.manifestUrl).href, 'release', token)
+    const { resultId, ...payload } = release
+    if (bytes.length !== file.bytes || await sha(bytes) !== file.sha256 || resultId !== id || await sha(new TextEncoder().encode(JSON.stringify(payload))) !== id) throw new CheckError('release-integrity-failed')
+    if (release.schemaVersion !== 2 || release.kind !== 'feed-quality-history' || release.operator !== c.operator || !Array.isArray(release.days) || release.days.length > 366) throw new CheckError('release-invalid')
+    readOperationalTime(`${release.from}T00:00:00Z`); readOperationalTime(`${release.to}T00:00:00Z`)
+    if (release.from > release.to) throw new CheckError('release-invalid')
+    const expected = studyDay(expectation.serviceDate)
+    const selected = release.days.filter(d => d.date === expectation.serviceDate)
+    if (release.from > expectation.serviceDate || release.to < expectation.serviceDate || selected.length === 0) return stage('degraded', 'consumer-period-overdue')
+    if (selected.length !== 1) throw new CheckError('release-invalid')
+    const day = selected[0]
+    if (day.status !== 'current') return stage('degraded', 'consumer-day-overdue')
+    if (!digest(day.revision) || !day.quality || day.day?.startUtc !== expected.startUtc || day.day?.endUtc !== expected.endUtc || day.day?.serviceDate !== expected.serviceDate || Date.parse(publishedAt) < Date.parse(expected.endUtc)) throw new CheckError('release-day-invalid')
+    return stage('healthy', 'consumer-release-verified', { lastSuccessAt: publishedAt, latestPeriodEnd: expected.endUtc }, { 'verified-release-bytes': bytes.length })
+  } catch (e) { return stage('degraded', `consumer-${e instanceof CheckError ? e.reason : 'document-invalid'}`) }
+}
+
+/** Portable Web-API checker; reads configured endpoints only, with no redirects or retries. */
+export async function observeFeeds(registryValue, configValue, { fetchImpl = fetch, clock = Date.now, tokens = {}, limits = OBSERVER_LIMITS } = {}) {
+  const registry = readFeedRegistry(registryValue), c = config(configValue, registry)
+  // Tests may lower bounds, never raise them.
+  for (const [key, value] of Object.entries(OBSERVER_LIMITS)) if (!Number.isSafeInteger(limits[key]) || limits[key] < 1 || limits[key] > value) throw new Error('Invalid check limit')
+  const api = client(fetchImpl, tokens, limits), startedAt = new Date(clock()).toISOString()
+  let source = null, sourceError = null
+  try { source = (await api.health(c.healthUrl)).json; refresh(registry, source, c.producerId, clock()) }
+  catch (e) { source = null; sourceError = e instanceof CheckError ? e.reason : 'document-invalid' }
+  const checks = []
+  for (const target of c.consumers) {
+    const definition = registry.feeds.find(f => f.id === target.feedId)
+    if (definition.state !== 'active') continue
+    const expectation = dailyFeedExpectation(target, clock())
+    const health = await consumer(target, expectation, api, clock(), registry.clockSkewSeconds, tokens.consumers?.[target.feedId])
+    checks.push({ feedId: target.feedId, expectation, ...health })
+  }
+  const finished = clock()
+  for (const check of checks) {
+    const target = c.consumers.find(t => t.feedId === check.feedId), latest = dailyFeedExpectation(target, finished)
+    if (latest.serviceDate !== check.expectation.serviceDate) Object.assign(check, stage('degraded', 'consumer-expectation-advanced'), { expectation: latest })
+  }
+  let report
+  if (source) report = refresh(registry, source, c.producerId, finished)
+  else report = recorderFeedHealth(registry, sourceError === 'document-invalid' ? {} : null, { producerId: c.producerId, now: finished })
+  if (report.telemetry.state === 'current') for (const check of checks) {
+    const feed = report.feeds.find(f => f.feedId === check.feedId)
+    const { feedId: _feedId, expectation: _expectation, ...health } = check
+    feed.stages = feed.stages.map(s => s.stageId === 'serve' ? health : s)
+    feed.state = summarizeFeedStages(feed.stages)
+  }
+  return { schemaVersion: 1, kind: 'feed-observer-check', startedAt, completedAt: new Date(finished).toISOString(), sourceError, requests: api.usage.requests, bytes: api.usage.bytes,
+    report: readFeedHealth(report, registry), consumers: checks }
+}
+
+/** Assess a stored check at read time. A successful old check is historical evidence. */
+export function assessObserverCheck(registryValue, checked, { now = Date.now(), maxAgeSeconds = 180 } = {}) {
+  const registry = readFeedRegistry(registryValue)
+  if (!Number.isFinite(now) || !Number.isSafeInteger(maxAgeSeconds) || maxAgeSeconds < 1 || maxAgeSeconds > 3600) throw new Error('Invalid observer clock/threshold')
+  const unknown = reason => ({ state: 'unknown', reasons: [reason] })
+  if (!checked) return unknown('checker-missing')
+  try {
+    if (checked.schemaVersion !== 1 || checked.kind !== 'feed-observer-check' || !Array.isArray(checked.consumers)) return unknown('checker-invalid')
+    const completed = Date.parse(readOperationalTime(checked.completedAt)), started = Date.parse(readOperationalTime(checked.startedAt))
+    if (started > completed || completed > now + registry.clockSkewSeconds * 1000) return unknown('checker-clock-invalid')
+    if (now - completed > maxAgeSeconds * 1000) return unknown('checker-stale')
+    const report = refresh(registry, checked.report, checked.report.producerId, now)
+    if (report.telemetry.state !== 'current') return unknown(`telemetry-${report.telemetry.state}`)
+    for (const c of checked.consumers) {
+      if (Date.parse(readOperationalTime(c.expectation.nextDeadlineAt)) <= now) return unknown('consumer-expectation-expired')
+      if (!['healthy', 'degraded', 'unknown', 'waiting'].includes(c.state)) return unknown('checker-invalid')
+    }
+    const states = [report.capacity.state, ...report.feeds.filter(f => f.configuredState === 'active').map(f => f.state), ...checked.consumers.map(c => c.state)]
+    if (states.includes('degraded')) return { state: 'degraded', reasons: ['feed-attention-required'] }
+    if (states.includes('unknown')) return unknown('feed-evidence-unavailable')
+    return { state: 'healthy', reasons: ['check-current'] }
+  } catch { return unknown('checker-invalid') }
+}

package/network-chunk-cli.d.mts

@@ -0,0 +1 @@
+export function runNetworkChunkCli(argv?: readonly string[]): Promise<void>

package/network-chunk-cli.mjs

@@ -0,0 +1,65 @@
+import { chunkNetworkSnapshot, extractNetworkWindow } from './network-chunks.mjs'
+
+import { createWriteStream } from 'node:fs'
+import { mkdir, readFile } from 'node:fs/promises'
+import { basename, dirname, extname, join, resolve } from 'node:path'
+
+function argument(argv, name, fallback) {
+  const index = argv.indexOf(`--${name}`)
+  return index === -1 ? fallback : argv[index + 1]
+}
+
+function parseClock(value) {
+  const [hours, minutes = 0] = value.split(':').map(Number)
+  if (!Number.isFinite(hours) || !Number.isFinite(minutes)) {
+    throw new Error(`Invalid clock value ${value}`)
+  }
+  return hours * 3600 + minutes * 60
+}
+
+async function writeJson(filePath, value) {
+  await mkdir(dirname(filePath), { recursive: true })
+  await new Promise((resolvePromise, rejectPromise) => {
+    const output = createWriteStream(filePath)
+    output.once('error', rejectPromise)
+    output.once('finish', resolvePromise)
+    output.end(JSON.stringify(value))
+  })
+}
+
+export async function runNetworkChunkCli(argv = process.argv.slice(2)) {
+  const input = argument(argv, 'input')
+  const manifestOutput = argument(argv, 'manifest')
+  const openingOutput = argument(argv, 'opening')
+  if (!input || !manifestOutput || !openingOutput) {
+    throw new Error(
+      'Usage: node scripts/chunk-network-snapshot.mjs --input day.json --manifest day-manifest.json --opening morning.json [--chunk-hours 2] [--opening-start 06:45] [--opening-end 08:45] [--focus 07:45]',
+    )
+  }
+  const snapshot = JSON.parse(await readFile(resolve(input), 'utf8'))
+  const manifestPath = resolve(manifestOutput)
+  const outputDirectory = dirname(manifestPath)
+  const outputStem = basename(manifestPath, extname(manifestPath)).replace(/-manifest$/, '')
+  const chunkDirectoryName = `${outputStem}-chunks`
+  const { manifest, chunks } = chunkNetworkSnapshot(
+    snapshot,
+    Number(argument(argv, 'chunk-hours', '2')) * 3600,
+    chunkDirectoryName,
+  )
+  const opening = extractNetworkWindow(
+    snapshot,
+    parseClock(argument(argv, 'opening-start', '06:45')),
+    parseClock(argument(argv, 'opening-end', '08:45')),
+    parseClock(argument(argv, 'focus', '07:45')),
+  )
+  await Promise.all([
+    writeJson(manifestPath, manifest),
+    writeJson(resolve(openingOutput), opening),
+    ...chunks.map(({ descriptor, payload }) =>
+      writeJson(join(outputDirectory, descriptor.path), payload),
+    ),
+  ])
+  console.log(
+    `Wrote ${manifestOutput}, ${chunks.length} progressive chunks and ${opening.trains.length} opening journeys.`,
+  )
+}

package/observation-windows.d.mts

@@ -0,0 +1,38 @@
+export interface ObservationRecord<T> {
+  readonly id: string
+  readonly timeUtc: string
+  readonly receivedAt?: string
+  readonly sourceRecordedAt?: string
+  readonly sourceRefs?: readonly string[]
+  readonly value: T
+}
+export interface ObservationWindowOptions {
+  readonly startUtc: string
+  readonly endUtc: string
+  readonly sourceId: string
+  readonly evidenceKind: string
+  readonly timeBasis: 'capture-schedule' | 'receipt' | 'measurement'
+  readonly windowSeconds?: number
+  readonly cadenceSeconds?: number
+  readonly minimumRecords?: number
+  readonly duplicatePolicy?: 'reject' | 'omit' | 'last'
+  readonly pathPrefix?: string
+  readonly maxRecords?: number
+  readonly maxBytes?: number
+}
+export interface ObservationCoverage {
+  inputCount: number; retainedRecords: number; duplicates: number; conflicts: number; outsideWindow: number;
+  occupiedSlots: number; expectedSlots: number; missingSlots: number[]; longestGapSeconds: number;
+  leadingGapSeconds: number; trailingGapSeconds: number;
+}
+export interface ObservationWindowDescriptor {
+  id: string; windowStart: number; windowEnd: number; path: string; frameCount: number; bytes: number; sha256: string;
+}
+export function jsonArtifact(value: unknown): { json: string; bytes: number; sha256: string }
+export function compileObservationWindows<T>(input: Iterable<ObservationRecord<T>>, options: ObservationWindowOptions): {
+  manifest: { schemaVersion: number; sourceId: string; evidenceKind: string; timeBasis: ObservationWindowOptions['timeBasis'];
+    startUtc: string; endUtc: string; durationSeconds: number; cadenceSeconds: number; duplicatePolicy: string;
+    coverage: ObservationCoverage; chunks: ObservationWindowDescriptor[] };
+  chunks: { descriptor: ObservationWindowDescriptor; artifact: { windowStart: number; windowEnd: number;
+    frames: (ObservationRecord<T> & { time: number })[] }; json: string }[];
+}

package/observation-windows.mjs

@@ -0,0 +1,76 @@
+import { createHash } from 'node:crypto'
+
+function instant(value) {
+  if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(value) || !Number.isFinite(Date.parse(value)) ||
+    new Date(`${value.slice(0, 10)}T00:00:00Z`).toISOString().slice(0, 10) !== value.slice(0, 10) || Number(value.slice(11, 13)) > 23) throw new Error('An explicit valid observation timestamp with timezone is required')
+  return Date.parse(value)
+}
+function positive(value, name) {
+  if (!Number.isSafeInteger(value) || value < 1) throw new Error(`Invalid ${name}`)
+}
+
+/** Hash the exact JSON bytes that are written and subsequently verified by consumers. */
+export function jsonArtifact(value) {
+  const json = `${JSON.stringify(value)}\n`
+  return { json, bytes: Buffer.byteLength(json), sha256: createHash('sha256').update(json).digest('hex') }
+}
+
+/** Bounded offline compilation. Clock semantics and evidence kind must be named by the caller. */
+export function compileObservationWindows(input, options) {
+  const { startUtc, endUtc, sourceId, evidenceKind, timeBasis, windowSeconds = 7200, cadenceSeconds = 60,
+    minimumRecords = 0, duplicatePolicy = 'reject', pathPrefix = 'observation-chunks',
+    maxRecords = 100000, maxBytes = 256 * 1024 * 1024 } = options
+  const start = instant(startUtc), end = instant(endUtc), durationSeconds = (end - start) / 1000
+  positive(windowSeconds, 'window duration'); positive(cadenceSeconds, 'cadence'); positive(maxRecords, 'record limit'); positive(maxBytes, 'byte limit')
+  if (typeof sourceId !== 'string' || !sourceId || typeof evidenceKind !== 'string' || !evidenceKind || !['capture-schedule', 'receipt', 'measurement'].includes(timeBasis) || end <= start ||
+    !Number.isSafeInteger(minimumRecords) || minimumRecords < 0 || !['reject', 'omit', 'last'].includes(duplicatePolicy) ||
+    typeof pathPrefix !== 'string' || !pathPrefix || pathPrefix.startsWith('/') || pathPrefix.split('/').some(part => part === '..' || !part)) throw new Error('Invalid observation archive options')
+  const records = new Map(), rejected = new Set()
+  let inputCount = 0, inputBytes = 0, duplicates = 0, conflicts = 0, outsideWindow = 0
+  for (const record of input) {
+    if (++inputCount > maxRecords) throw new Error('Observation record limit exceeded')
+    const serialized = JSON.stringify(record)
+    if (serialized === undefined || (inputBytes += Buffer.byteLength(serialized)) > maxBytes) throw new Error('Observation byte limit exceeded')
+    if (!record || typeof record.id !== 'string' || !record.id || record.value === undefined) throw new Error('Observation identity and value required')
+    const timestamp = instant(record.timeUtc)
+    if (record.receivedAt !== undefined) instant(record.receivedAt)
+    if (record.sourceRecordedAt !== undefined) instant(record.sourceRecordedAt)
+    if (timestamp < start || timestamp >= end) { outsideWindow++; continue }
+    if (rejected.has(record.id)) { conflicts++; continue }
+    const prior = records.get(record.id)
+    if (prior) {
+      if (prior.serialized === serialized) { duplicates++; continue }
+      conflicts++
+      if (duplicatePolicy === 'reject') throw new Error(`Conflicting observation identity: ${record.id}`)
+      if (duplicatePolicy === 'omit') { records.delete(record.id); rejected.add(record.id); continue }
+    }
+    records.set(record.id, { record, serialized, timestamp })
+  }
+  const ordered = [...records.values()].sort((a, b) => a.timestamp - b.timestamp || a.record.id.localeCompare(b.record.id))
+  if (ordered.length < minimumRecords) throw new Error(`Observation archive has ${ordered.length} records; ${minimumRecords} required`)
+  const slots = new Set(ordered.map(item => Math.floor((item.timestamp - start) / (cadenceSeconds * 1000))))
+  const expectedSlots = Math.ceil(durationSeconds / cadenceSeconds)
+  const missingSlots = []
+  // A caller-controlled duration/cadence must not allocate an unbounded coverage index.
+  if (expectedSlots > maxRecords) throw new Error('Observation coverage index limit exceeded')
+  for (let slot = 0; slot < expectedSlots; slot++) if (!slots.has(slot)) missingSlots.push(slot)
+  let longestGapSeconds = 0
+  for (let i = 1; i < ordered.length; i++) longestGapSeconds = Math.max(longestGapSeconds, (ordered[i].timestamp - ordered[i - 1].timestamp) / 1000)
+  const coverage = { inputCount, retainedRecords: ordered.length, duplicates, conflicts, outsideWindow,
+    occupiedSlots: slots.size, expectedSlots, missingSlots, longestGapSeconds,
+    leadingGapSeconds: ordered.length ? (ordered[0].timestamp - start) / 1000 : durationSeconds,
+    trailingGapSeconds: ordered.length ? (end - ordered.at(-1).timestamp) / 1000 : durationSeconds }
+  const groups = new Map()
+  for (const { record, timestamp } of ordered) {
+    const index = Math.floor((timestamp - start) / (windowSeconds * 1000))
+    if (!groups.has(index)) groups.set(index, [])
+    groups.get(index).push({ ...record, time: (timestamp - start) / 1000 })
+  }
+  const chunks = [...groups].map(([index, frames]) => {
+    const windowStart = index * windowSeconds, windowEnd = Math.min(durationSeconds, windowStart + windowSeconds)
+    const id = `${windowStart}-${windowEnd}`, artifact = { windowStart, windowEnd, frames }, encoded = jsonArtifact(artifact)
+    return { descriptor: { id, windowStart, windowEnd, path: `${pathPrefix}/${id}.json`, frameCount: frames.length, bytes: encoded.bytes, sha256: encoded.sha256 }, artifact, json: encoded.json }
+  })
+  return { manifest: { schemaVersion: 1, sourceId, evidenceKind, timeBasis, startUtc, endUtc, durationSeconds, cadenceSeconds,
+    duplicatePolicy, coverage, chunks: chunks.map(chunk => chunk.descriptor) }, chunks }
+}

package/operational-files.mjs

@@ -0,0 +1,21 @@
+import { open, constants } from 'node:fs/promises'
+
+/** Read a bounded regular file without following a final symlink or blocking on a FIFO. */
+export async function readOperationalFile(path, limit) {
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
+  try {
+    const stat = await handle.stat()
+    if (!stat.isFile() || stat.size > limit) throw new Error('Input must be a bounded regular file')
+    // Size the buffer from the file, not the limit, so a generous limit costs nothing for small inputs; one spare byte detects growth.
+    const bytes = Buffer.alloc(Math.min(limit, stat.size) + 1)
+    let size = 0
+    while (size <= limit) {
+      const { bytesRead } = await handle.read(bytes, size, bytes.length - size, null)
+      if (!bytesRead) break
+      size += bytesRead
+    }
+    const after = await handle.stat()
+    if (size > limit || size !== stat.size || after.size !== stat.size || after.mtimeMs !== stat.mtimeMs) throw new Error('Input changed or exceeds read limit')
+    return bytes.subarray(0, size)
+  } finally { await handle.close() }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@motionstudies/data",
-  "version": "0.1.0-alpha.3",
+  "version": "0.1.0-alpha.30",
   "private": false,
   "type": "module",
   "description": "Offline data preparation primitives for Motion Studies.",
@@ -23,6 +23,18 @@
     "node": ">=24.0.0"
   },
   "exports": {
+    "./observation-windows": {
+      "types": "./observation-windows.d.mts",
+      "import": "./observation-windows.mjs"
+    },
+    "./adsb-heatmap": {
+      "types": "./adsb-heatmap.d.mts",
+      "import": "./adsb-heatmap.mjs"
+    },
+    "./air-endpoints": {
+      "types": "./air-endpoints.d.mts",
+      "import": "./air-endpoints.mjs"
+    },
     "./gtfs": {
       "types": "./gtfs.d.mts",
       "import": "./gtfs.mjs"
@@ -38,6 +50,74 @@
     "./merge-network": {
       "types": "./merge-network.d.mts",
       "import": "./merge-network.mjs"
+    },
+    "./network-chunk-cli": {
+      "types": "./network-chunk-cli.d.mts",
+      "import": "./network-chunk-cli.mjs"
+    },
+    "./source-store": {
+      "types": "./source-store.d.mts",
+      "import": "./source-store.mjs"
+    },
+    "./release": {
+      "types": "./release.d.mts",
+      "import": "./release.mjs"
+    },
+    "./release-files": {
+      "types": "./release-files.d.mts",
+      "import": "./release-files.mjs"
+    },
+    "./uk-service-day": {
+      "types": "./uk-service-day.d.mts",
+      "import": "./uk-service-day.mjs"
+    },
+    "./webtris": {
+      "types": "./webtris.d.mts",
+      "import": "./webtris.mjs"
+    },
+    "./rail-wtt": {
+      "types": "./rail-wtt.d.mts",
+      "import": "./rail-wtt.mjs"
+    },
+    "./rail-journeys": {
+      "types": "./rail-journeys.d.mts",
+      "import": "./rail-journeys.mjs"
+    },
+    "./rail-routing": {
+      "types": "./rail-routing.d.mts",
+      "import": "./rail-routing.mjs"
+    },
+    "./rail-public-calls": {
+      "types": "./rail-public-calls.d.mts",
+      "import": "./rail-public-calls.mjs"
+    },
+    "./rail-geometry": {
+      "types": "./rail-geometry.d.mts",
+      "import": "./rail-geometry.mjs"
+    },
+    "./feed-observability": {
+      "types": "./feed-observability.d.mts",
+      "import": "./feed-observability.mjs"
+    },
+    "./recorder-observability": {
+      "types": "./recorder-observability.d.mts",
+      "import": "./recorder-observability.mjs"
+    },
+    "./recorder-evidence": {
+      "types": "./recorder-evidence.d.mts",
+      "import": "./recorder-evidence.mjs"
+    },
+    "./feed-incidents": {
+      "types": "./feed-incidents.d.mts",
+      "import": "./feed-incidents.mjs"
+    },
+    "./feed-observer": {
+      "types": "./feed-observer.d.mts",
+      "import": "./feed-observer.mjs"
+    },
+    "./feed-expectations": {
+      "types": "./feed-expectations.d.mts",
+      "import": "./feed-expectations.mjs"
     }
   },
   "files": [
@@ -46,11 +126,14 @@
     "**/*.d.ts",
     "**/*.d.mts",
     "**/*.css",
+    "**/*.woff2",
+    "**/*-OFL.txt",
     "README.md",
     "LICENSE"
   ],
   "sideEffects": false,
   "dependencies": {
-    "@motionstudies/core": "0.1.0-alpha.3"
+    "@motionstudies/core": "0.1.0-alpha.30",
+    "fast-xml-parser": "5.11.1"
   }
 }

package/rail-geometry.d.mts

@@ -0,0 +1,7 @@
+import type {RailCoordinate, RailOsm} from './rail-routing.mjs'
+export type RailStop = [number, number, string, string, string]
+export interface CorridorGeometry { stops: RailStop[]; paths: RailCoordinate[][]; wayIds?: number[]; pairPaths?: Record<string, RailCoordinate[]>; corridorPaths?: RailCoordinate[][] }
+export interface CorridorJourney { id: string; direction: 'inbound' | 'outbound'; stops: [number, number, number][] }
+export function distance(a: RailCoordinate, b: RailCoordinate): number
+export function railCorridorGeometry(osm: RailOsm, stationCodes: string[]): CorridorGeometry & {wayIds: number[]}
+export function assembleRailCorridor<T extends CorridorJourney, M extends {coverage?: Record<string, unknown>}, B> (input: {journeys: T[]; geometry: CorridorGeometry; metadata: M; bounds: B; excluded?: unknown[]}): {metadata: M & {coverage: {includedJourneys: number; excluded: unknown[]}}; bounds: B; stops: RailStop[]; paths: RailCoordinate[][]; edges: never[]; trains: (T & {start: number; end: number; pathSegments: number[]})[]; corridorPaths: RailCoordinate[][]; fadeKilometres: number}

package/rail-geometry.mjs

@@ -0,0 +1,108 @@
+export const distance = (a, b) => Math.hypot((a[0] - b[0]) * Math.cos((a[1] + b[1]) * Math.PI / 360), a[1] - b[1]) * 111.32
+/** Route on connected OSM rails through each station; never replace gaps with straight lines. */
+export function railCorridorGeometry(osm, stationCodes) {
+  if (!Array.isArray(stationCodes) || stationCodes.length < 2 || new Set(stationCodes).size !== stationCodes.length) throw new Error('Expected distinct corridor station codes')
+  const nodes = new Map(osm.elements.filter(e => e.type === 'node').map(e => [e.id, e]))
+  const graph = new Map()
+  for (const way of osm.elements.filter(e => e.type === 'way' && e.tags?.railway === 'rail' && !['yard', 'siding', 'spur'].includes(e.tags.service))) {
+    for (let i = 1; i < way.nodes.length; i++) {
+      const a = nodes.get(way.nodes[i - 1]), b = nodes.get(way.nodes[i])
+      if (!a || !b) throw new Error(`Missing OSM nodes on ${way.id}`)
+      const weight = distance([a.lon, a.lat], [b.lon, b.lat])
+      for (const [from, to] of [[a.id, b.id], [b.id, a.id]]) {
+        if (!graph.has(from)) graph.set(from, [])
+        graph.get(from).push({ to, weight, way: way.id })
+      }
+    }
+  }
+  const stations = stationCodes.map(code => {
+    const station = [...nodes.values()].find(node => node.tags?.railway === 'station' && node.tags['ref:crs'] === code)
+    if (!station) throw new Error(`Missing OSM station ${code}`)
+    return station
+  })
+  // The largest connected component excludes isolated rails and works-in-progress.
+  const visited = new Set(); let component = []
+  for (const start of graph.keys()) {
+    if (visited.has(start)) continue
+    const found = [], queue = [start]; visited.add(start)
+    for (let i = 0; i < queue.length; i++) {
+      const id = queue[i]; found.push(id)
+      for (const { to } of graph.get(id)) if (!visited.has(to)) { visited.add(to); queue.push(to) }
+    }
+    if (found.length > component.length) component = found
+  }
+  if (!component.length) throw new Error('No connected railway geometry')
+  const anchors = stations.map(station => {
+    const point = [station.lon, station.lat]
+    const id = component.reduce((best, id) => distance(point, [nodes.get(id).lon, nodes.get(id).lat]) < distance(point, [nodes.get(best).lon, nodes.get(best).lat]) ? id : best, component[0])
+    if (distance(point, [nodes.get(id).lon, nodes.get(id).lat]) > 0.25) throw new Error(`Station too far from connected railway: ${station.tags.name}`)
+    return id
+  })
+  const queue = [{ id: anchors[0], weight: 0 }], best = new Map([[anchors[0], 0]]), previous = new Map()
+  const target = anchors.at(-1)
+  while (queue.length) {
+    queue.sort((a, b) => b.weight - a.weight)
+    const current = queue.pop()
+    if (current.weight !== best.get(current.id)) continue
+    if (current.id === target) break
+    for (const edge of graph.get(current.id)) {
+      const weight = current.weight + edge.weight
+      if (weight >= (best.get(edge.to) ?? Infinity)) continue
+      best.set(edge.to, weight); previous.set(edge.to, { id: current.id, way: edge.way }); queue.push({ id: edge.to, weight })
+    }
+  }
+  if (!best.has(target)) throw new Error('Disconnected railway corridor')
+  const ids = [target], wayIds = new Set()
+  while (ids.at(-1) !== anchors[0]) {
+    const edge = previous.get(ids.at(-1)); wayIds.add(edge.way); ids.push(edge.id)
+  }
+  const line = ids.reverse().map(id => [nodes.get(id).lon, nodes.get(id).lat])
+  // Snap every intermediate station to this continuous route, avoiding jumps between parallel tracks.
+  const indexes = stations.map(station => line.reduce((best, point, i) => distance(point, [station.lon, station.lat]) < distance(line[best], [station.lon, station.lat]) ? i : best, 0))
+  const paths = []
+  for (let i = 0; i < indexes.length; i++) {
+    if (distance(line[indexes[i]], [stations[i].lon, stations[i].lat]) > 0.25) throw new Error(`Railway misses ${stationCodes[i]}`)
+    if (!i) continue
+    if (indexes[i] <= indexes[i - 1]) throw new Error(`Railway reverses at ${stationCodes[i]}`)
+    const path = line.slice(indexes[i - 1], indexes[i] + 1)
+    const length = path.slice(1).reduce((sum, point, j) => sum + distance(path[j], point), 0)
+    if (length > Math.max(2, distance(path[0], path.at(-1)) * 1.8)) throw new Error(`Implausible railway detour to ${stationCodes[i]}`)
+    paths.push(path)
+  }
+  return { stops: stations.map((station, i) => [...line[indexes[i]], station.tags.name, '', `crs:${stationCodes[i]}`]), paths, wayIds: [...wayIds].sort((a, b) => a - b) }
+}
+
+
+/** Shared output boundary for operator tables or normalized machine-readable schedules. */
+export function assembleRailCorridor({ journeys, geometry, metadata, bounds, excluded = [] }) {
+  const ids = new Set(), paths = [], pathIndexes = new Map()
+  const trains = journeys.map(train => {
+    if (ids.has(train.id)) throw new Error(`Duplicate rail identity: ${train.id}`)
+    ids.add(train.id)
+    if (train.stops.length < 2 || !['inbound', 'outbound'].includes(train.direction)) throw new Error(`Invalid rail journey: ${train.id}`)
+    for (let i = 0; i < train.stops.length; i++) {
+      const [stop, a, d] = train.stops[i]
+      if (!geometry.stops[stop] || !Number.isFinite(a) || !Number.isFinite(d) || d < a || (i && a < train.stops[i - 1][2])) throw new Error(`Non-monotonic rail journey: ${train.id}`)
+    }
+    return { ...train, start: train.stops[0][1], end: train.stops.at(-1)[2], pathSegments: train.stops.slice(1).map(([to], index) => {
+      const from = train.stops[index][0], key = `${from}:${to}`
+      if (from === to) throw new Error(`Repeated corridor station: ${train.id}`)
+      if (pathIndexes.has(key)) return pathIndexes.get(key)
+      if (geometry.pairPaths) {
+        const path = geometry.pairPaths[key]
+        if (!path || path.length < 2 || JSON.stringify(path[0]) !== JSON.stringify(geometry.stops[from].slice(0, 2)) || JSON.stringify(path.at(-1)) !== JSON.stringify(geometry.stops[to].slice(0, 2))) throw new Error(`Missing or disconnected rail path: ${key}`)
+        const pathIndex = paths.length
+        paths.push(path); pathIndexes.set(key, pathIndex)
+        return pathIndex
+      }
+      const segments = geometry.paths.slice(Math.min(from, to), Math.max(from, to))
+      if (segments.length !== Math.abs(to - from) || segments.some(path => path.length < 2)) throw new Error(`Missing corridor geometry: ${key}`)
+      for (let i = 1; i < segments.length; i++) if (JSON.stringify(segments[i - 1].at(-1)) !== JSON.stringify(segments[i][0])) throw new Error(`Disconnected corridor geometry: ${key}`)
+      const path = segments.flatMap((points, i) => i ? points.slice(1) : points)
+      const pathIndex = paths.length
+      paths.push(from < to ? path : [...path].reverse()); pathIndexes.set(key, pathIndex)
+      return pathIndex
+    }) }
+  }).sort((a, b) => a.start - b.start || a.id.localeCompare(b.id))
+  return { metadata: { ...metadata, coverage: { ...metadata.coverage, includedJourneys: trains.length, excluded } }, bounds, stops: geometry.stops, paths, edges: [], trains, corridorPaths: geometry.corridorPaths ?? geometry.paths, fadeKilometres: 4 }
+}