@motionstudies/data 0.1.0-alpha.3 → 0.1.0-alpha.31

package/air-endpoints.mjs

@@ -0,0 +1,148 @@
+import { readFile, readdir, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { gunzipSync } from 'node:zlib'
+import { createHash } from 'node:crypto'
+import { isAirContinent } from '@motionstudies/core/air-continents'
+import { parseCsvLine } from './gtfs.mjs'
+import { decodeAdsbHeatmap } from './adsb-heatmap.mjs'
+
+export function parseAirports(csv, { continentOverrides = [] } = {}) {
+  const [header, ...lines] = csv.trim().split(/\r?\n/)
+  const keys = parseCsvLine(header)
+  const rows = lines.map(line => Object.fromEntries(parseCsvLine(line).map((value, index) => [keys[index], value])))
+  if (!Array.isArray(continentOverrides)) throw new Error('Continent overrides must be an array')
+  const identifiers = new Set(rows.map(row => row.ident)), overrides = new Map()
+  for (const entry of continentOverrides) {
+    if (!entry || !identifiers.has(entry.airportIdent) || !isAirContinent(entry.continent)
+      || typeof entry.reason !== 'string' || !entry.reason.trim() || overrides.has(entry.airportIdent)) {
+      throw new Error('Each continent override needs a unique reference airportIdent, valid continent and reason')
+    }
+    overrides.set(entry.airportIdent, entry.continent)
+  }
+  const geography = row => {
+    const continent = overrides.get(row.ident) ?? row.continent?.trim().toUpperCase()
+    if (!continent) return {}
+    if (!isAirContinent(continent)) throw new Error(`Invalid continent for airport ${row.ident}: ${continent}`)
+    return { continent, continentSource: overrides.has(row.ident) ? 'override' : 'ourairports' }
+  }
+  return rows
+    .filter(row => ['large_airport', 'medium_airport'].includes(row.type) && row.ident && row.elevation_ft)
+    .map(row => ({ icao: row.ident, iata: row.iata_code, name: row.name, city: row.municipality,
+      longitude: Number(row.longitude_deg), latitude: Number(row.latitude_deg), elevation: Number(row.elevation_ft), ...geography(row) }))
+    .filter(row => [row.longitude, row.latitude, row.elevation].every(Number.isFinite))
+}
+
+function distance(sample, airport) {
+  const longitude = ((sample[1] - airport.longitude + 540) % 360) - 180
+  return Math.hypot(longitude * 111.32 * Math.cos((sample[2] + airport.latitude) * Math.PI / 360), (sample[2] - airport.latitude) * 111.32)
+}
+
+function atAirport(sample, airport) {
+  return sample[3] >= airport.elevation - 500 && sample[3] <= airport.elevation + 1500 && sample[4] <= 250 && distance(sample, airport) <= 6
+}
+
+/** Conservative endpoint association; cruise-only traces and ambiguous neighbouring airports stay unknown. */
+export function inferAirEndpoints(samples, airports) {
+  if (samples.length < 4) return {}
+  const maximumAltitude = Math.max(...samples.map(sample => sample[3]))
+  const endpoint = (arrival) => {
+    const ordered = arrival ? [...samples].reverse() : samples
+    const candidates = airports.filter(airport => atAirport(ordered[0], airport) && maximumAltitude >= airport.elevation + 3000)
+    if (candidates.length !== 1) return undefined
+    const airport = candidates[0]
+    let boundary = ordered[0]
+    for (const sample of ordered.slice(1)) {
+      if (!atAirport(sample, airport)) break
+      boundary = sample
+    }
+    return { icao: airport.icao, iata: airport.iata, name: airport.name, city: airport.city, time: boundary[0], evidence: 'observed-endpoint',
+      ...(isAirContinent(airport.continent) ? { continent: airport.continent, continentSource: airport.continentSource ?? 'ourairports' } : {}) }
+  }
+  const origin = endpoint(false), destination = endpoint(true)
+  if (origin && destination && (origin.icao === destination.icao || origin.time >= destination.time)) return {}
+  return { ...(origin ? { origin } : {}), ...(destination ? { destination } : {}) }
+}
+
+function segmentsFor(samples) {
+  const segments = []
+  let current = []
+  for (const sample of samples.sort((a, b) => a[0] - b[0])) {
+    const previous = current.at(-1)
+    if (previous && previous[0] === sample[0]) continue
+    if (previous && (sample[0] - previous[0] > 1800 || sample[5] !== previous[5])) {
+      segments.push(current)
+      current = []
+    }
+    current.push(sample)
+  }
+  if (current.length) segments.push(current)
+  return segments
+}
+
+/** Enrich an existing study and its chunks using cached, same-date global heatmaps. No network requests. */
+export async function enrichAirEndpoints({ manifestPath, snapshotPaths = [], heatmapDirectory, airportCsvPath, utcOffsetHours }) {
+  const readJson = async path => JSON.parse(await readFile(path, 'utf8'))
+  const manifest = await readJson(manifestPath)
+  const snapshots = await Promise.all(snapshotPaths.map(readJson))
+  const date = manifest.metadata.serviceDate
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(utcOffsetHours)) throw new Error('A service date and UTC offset are required')
+  if (snapshots.some(snapshot => snapshot.metadata.serviceDate !== date)) throw new Error('Air snapshots must share the manifest service date')
+  const tracks = [...manifest.aircraft, ...snapshots.flatMap(snapshot => snapshot.tracks)]
+  const wanted = new Set(tracks.map(track => parseInt(track.icaoAddress ?? track.id.split('-')[0], 16)))
+  const records = new Map()
+  const files = (await readdir(heatmapDirectory)).filter(file => file.endsWith('.bin.ttf')).sort()
+  if (!files.length) throw new Error('No cached heatmap slices found')
+  const sources = []
+  for (const file of files) {
+    const raw = await readFile(join(heatmapDirectory, file))
+    const used = decodeAdsbHeatmap(gunzipSync(raw), {
+      serviceDate: date, utcOffsetHours, windowStart: 0, windowEnd: 86400, addresses: wanted,
+    }, (id, sample) => {
+      const samples = records.get(id) ?? []
+      sample[5] ??= samples.at(-1)?.[5] ?? ''
+      samples.push(sample)
+      records.set(id, samples)
+    })
+    if (used) sources.push({ file, sha256: createHash('sha256').update(raw).digest('hex') })
+  }
+  const csv = await readFile(airportCsvPath, 'utf8')
+  if (!sources.length) throw new Error('No same-date observations matched the study aircraft')
+  const airports = parseAirports(csv)
+  if (!airports.length) throw new Error('No usable airports in the reference CSV')
+  const segments = new Map([...records].map(([id, samples]) => [id, segmentsFor(samples).map(samples => ({
+    start: samples[0][0], end: samples.at(-1)[0], callsign: samples[0][5], ...inferAirEndpoints(samples, airports),
+  }))]))
+  const enrich = (track) => {
+    const candidates = (segments.get(parseInt(track.icaoAddress ?? track.id.split('-')[0], 16)) ?? [])
+      .filter(segment => segment.callsign && segment.callsign === track.callsign && segment.start <= track.start + 30 && segment.end >= track.end - 30)
+    const { origin: _origin, destination: _destination, ...rest } = track
+    if (candidates.length !== 1) return rest
+    const { origin, destination } = candidates[0]
+    return { ...rest, ...(origin ? { origin } : {}), ...(destination ? { destination } : {}) }
+  }
+  const metadata = {
+    serviceDate: date, method: 'observed-endpoint', note: 'Origin/destination inferred from full-trace endpoints within 6 km and 1500 ft above an airport; observed times, not schedules or confirmed flight plans.',
+    airportSource: 'https://ourairports.com/data/', airportLicense: 'Public Domain', airportSha256: createHash('sha256').update(csv).digest('hex'),
+    traceSource: manifest.metadata.sourceUrl, traceLicense: 'ODbL 1.0', utcOffsetHours, sourceFiles: sources,
+  }
+  const aircraft = manifest.aircraft.map(enrich)
+  const byId = new Map(aircraft.map(track => [track.id, track]))
+  const writes = []
+  for (const descriptor of manifest.chunks) {
+    const path = join(dirname(manifestPath), descriptor.path)
+    const chunk = await readJson(path)
+    chunk.tracks = chunk.tracks.map(track => {
+      const { origin: _origin, destination: _destination, ...rest } = track
+      const full = byId.get(track.id)
+      return { ...rest, ...(full?.origin ? { origin: full.origin } : {}), ...(full?.destination ? { destination: full.destination } : {}) }
+    })
+    const bytes = Buffer.from(`${JSON.stringify(chunk)}\n`)
+    if ('bytes' in descriptor) descriptor.bytes = bytes.length
+    if ('sha256' in descriptor) descriptor.sha256 = createHash('sha256').update(bytes).digest('hex')
+    writes.push([path, chunk])
+  }
+  writes.push([manifestPath, { ...manifest, metadata: { ...manifest.metadata, routeEnrichment: metadata }, aircraft }])
+  snapshots.forEach((snapshot, index) => writes.push([snapshotPaths[index], { ...snapshot, metadata: { ...snapshot.metadata, routeEnrichment: metadata }, tracks: snapshot.tracks.map(enrich) }]))
+  for (const [path, value] of writes) await writeFile(path, `${JSON.stringify(value)}\n`)
+  return { tracks: aircraft.length, origins: aircraft.filter(track => track.origin).length, destinations: aircraft.filter(track => track.destination).length }
+}

package/feed-expectations.d.mts

@@ -0,0 +1,3 @@
+export interface DailyExpectation { serviceDate: string; deadlineAt: string; nextDeadlineAt: string }
+/** Repeated local minutes use their later occurrence; missing minutes use the next valid minute. */
+export function dailyFeedExpectation(schedule: { timeZone: string; deadline: string }, now?: number): DailyExpectation

package/feed-expectations.mjs

@@ -0,0 +1,33 @@
+import { readOperationalTime } from './feed-observability.mjs'
+
+const shift = (date, days) => new Date(Date.parse(`${date}T00:00:00Z`) + days * 86400000).toISOString().slice(0, 10)
+function formatter(timeZone) {
+  return new Intl.DateTimeFormat('en-CA', { timeZone, year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hourCycle: 'h23' })
+}
+function wall(format, ms) {
+  const p = Object.fromEntries(format.formatToParts(ms).map(p => [p.type, p.value]))
+  return `${p.year}-${p.month}-${p.day}T${p.hour}:${p.minute}`
+}
+/** Latest occurrence of a repeated local minute; first valid minute after a DST gap. */
+function deadlineAt(date, deadline, format) {
+  const target = `${date}T${deadline}`, nominal = Date.parse(`${target}:00Z`)
+  let exact = null, next = null
+  for (let ms = nominal - 18 * 3600000; ms <= nominal + 18 * 3600000; ms += 60000) {
+    const local = wall(format, ms)
+    if (local === target) exact = ms
+    if (local.slice(0, 10) === date && local > target && (!next || local < next.local)) next = { local, ms }
+  }
+  const value = exact ?? next?.ms
+  if (value === undefined) throw new Error('Local deadline does not exist on this date')
+  return value
+}
+/** Latest completed service date whose following-day deadline has passed. */
+export function dailyFeedExpectation({ timeZone, deadline }, now = Date.now()) {
+  if (typeof timeZone !== 'string' || timeZone.length > 80 || !/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(deadline)) throw new Error('Invalid daily expectation')
+  readOperationalTime(new Date(now).toISOString())
+  const format = formatter(timeZone), today = wall(format, now).slice(0, 10)
+  const todayDeadline = deadlineAt(today, deadline, format), dueToday = now >= todayDeadline
+  const dueDate = dueToday ? today : shift(today, -1)
+  return { serviceDate: shift(dueDate, -1), deadlineAt: new Date(dueToday ? todayDeadline : deadlineAt(dueDate, deadline, format)).toISOString(),
+    nextDeadlineAt: new Date(dueToday ? deadlineAt(shift(today, 1), deadline, format) : todayDeadline).toISOString() }
+}

package/feed-incidents.d.mts

@@ -0,0 +1,44 @@
+import type { FeedHealth, OperationalState } from './feed-observability.mjs'
+export interface IncidentPolicy { openAfterSeconds: number; recoverAfterSeconds: number; retentionDays: number; maxEvents: number }
+export declare const INCIDENT_POLICY: Readonly<IncidentPolicy>
+export interface IncidentEvent {
+  id: string
+  incidentId: string
+  key: string
+  feedId: string | null
+  stageId: string
+  firstDetectedAt: string
+  at: string
+  transition: 'opened' | 'updated' | 'recovered'
+  state: OperationalState
+  reasons: string[]
+}
+export interface ActiveIncident {
+  incidentId: string
+  key: string
+  feedId: string | null
+  stageId: string
+  firstDetectedAt: string
+  openedAt: string
+  lastObservedAt: string
+  state: OperationalState
+  reasons: string[]
+  recoveryStartedAt?: string
+  recoveryProducerAt?: string
+}
+export interface IncidentState {
+  schemaVersion: 1
+  kind: 'feed-incident-state'
+  registryId: string
+  producerId: string
+  policy: IncidentPolicy
+  lastObservedAt: string
+  lastReportId: string
+  report: FeedHealth
+  active: Record<string, ActiveIncident>
+  pending: Record<string, { since: string }>
+  events: IncidentEvent[]
+  prunedEvents: number
+}
+export function readIncidentStore(root: string): Promise<IncidentState>
+export function updateIncidentStore(root: string, registry: unknown, report: unknown, options?: { policy?: Partial<IncidentPolicy>; now?: number }): Promise<{ state: IncidentState; changed: boolean; events: IncidentEvent[] }>

package/feed-incidents.mjs

@@ -0,0 +1,148 @@
+import { createHash, randomUUID } from 'node:crypto'
+import { mkdir, lstat, open, rename, rm } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { hostname } from 'node:os'
+import { readFeedRegistry, readFeedHealth, readOperationalTime } from './feed-observability.mjs'
+import { readOperationalFile } from './operational-files.mjs'
+
+const MAX_BYTES = 8 * 1024 * 1024
+export const INCIDENT_POLICY = Object.freeze({ openAfterSeconds: 120, recoverAfterSeconds: 60, retentionDays: 180, maxEvents: 10000 })
+const canonical = value => JSON.stringify(value, (_, v) => v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.keys(v).sort().map(k => [k, v[k]])) : v)
+const hash = value => createHash('sha256').update(value).digest('hex')
+const notInstrumented = new Set(['upload-evidence-unavailable', 'stage-evidence-unavailable', 'capture-threshold-unconfigured', 'normalization-threshold-unconfigured', 'analytics-threshold-unconfigured'])
+function policy(value) {
+  const p = { ...INCIDENT_POLICY, ...value }
+  if (Object.keys(p).some(k => !Object.hasOwn(INCIDENT_POLICY, k)) || Object.values(p).some(n => !Number.isSafeInteger(n) || n < 0) || p.openAfterSeconds > 86400 || p.recoverAfterSeconds > 86400 || p.retentionDays < 1 || p.retentionDays > 3650 || p.maxEvents < 1 || p.maxEvents > 20000) throw new Error('Invalid incident policy')
+  return p
+}
+function validateState(state) {
+  if (!state || state.schemaVersion !== 1 || state.kind !== 'feed-incident-state' || typeof state.registryId !== 'string' || typeof state.producerId !== 'string') throw new Error('Invalid incident state')
+  policy(state.policy)
+  readOperationalTime(state.lastObservedAt)
+  if (!/^[a-f0-9]{64}$/.test(state.lastReportId) || !Array.isArray(state.events) || state.events.length > state.policy.maxEvents) throw new Error('Invalid incident cursor/history')
+  for (const field of ['active', 'pending']) if (!state[field] || typeof state[field] !== 'object' || Array.isArray(state[field]) || Object.keys(state[field]).length > 4096) throw new Error('Invalid incident index')
+  if (!Number.isSafeInteger(state.prunedEvents) || state.prunedEvents < 0) throw new Error('Invalid retention counter')
+  for (const event of state.events) {
+    if (!/^[a-f0-9]{64}$/.test(event.id) || !/^[a-f0-9]{64}$/.test(event.incidentId) || !['opened', 'updated', 'recovered'].includes(event.transition)) throw new Error('Invalid incident event')
+    readOperationalTime(event.at)
+  }
+  if (new Set(state.events.map(e => e.id)).size !== state.events.length) throw new Error('Duplicate incident event')
+  return state
+}
+/** Reads the last atomically committed state; corruption never initializes a new history. */
+export async function readIncidentStore(root) {
+  const wrapper = JSON.parse(await readOperationalFile(join(root, 'state.json'), MAX_BYTES))
+  if (wrapper.sha256 !== hash(canonical(wrapper.state))) throw new Error('Incident state checksum mismatch')
+  return validateState(wrapper.state)
+}
+function conditions(report, registry) {
+  const entries = [], add = (feedId, stageId, disposition, state, reasons) => {
+    entries.push({ key: JSON.stringify([report.producerId, feedId, stageId]), feedId, stageId, disposition, state, reasons: [...reasons].sort() })
+  }
+  const available = report.telemetry.state === 'current'
+  add(null, 'telemetry', available ? 'healthy' : 'problem', available ? 'healthy' : 'unknown', available ? [] : [`telemetry-${report.telemetry.state}`])
+  add(null, 'capacity', !available ? 'hold' : report.capacity.state === 'healthy' ? 'healthy' : 'problem', report.capacity.state, report.capacity.reasons)
+  for (const feed of report.feeds) {
+    const definition = registry.feeds.find(f => f.id === feed.feedId)
+    const actionable = stage => stage.state === 'degraded' || stage.state === 'unknown' && !stage.reasons.some(r => notInstrumented.has(r))
+    const blocked = id => definition.stages.find(s => s.id === id).dependsOn.some(parent => actionable(feed.stages.find(s => s.stageId === parent)) || blocked(parent))
+    for (const stage of feed.stages) {
+      let disposition = !available || feed.configuredState !== 'active' ? 'hold' : stage.state === 'healthy' ? 'healthy' : actionable(stage) ? 'problem' : 'hold'
+      if (disposition === 'problem' && (blocked(stage.stageId) || stage.reasons.includes('host-storage-blocked'))) disposition = 'hold'
+      add(feed.feedId, stage.stageId, disposition, stage.state, stage.reasons)
+    }
+  }
+  return entries
+}
+function advance(previous, report, registry, p) {
+  const at = report.observedAt, now = Date.parse(at), reportId = hash(canonical(report))
+  if (previous) {
+    if (previous.registryId !== registry.id || previous.producerId !== report.producerId || canonical(previous.policy) !== canonical(p)) throw new Error('Store identity/policy mismatch')
+    const order = now - Date.parse(previous.lastObservedAt)
+    if (order < 0 || order === 0 && previous.lastReportId !== reportId) throw new Error('Out-of-order or conflicting observation')
+    if (order === 0) return { state: previous, changed: false, events: [] }
+  }
+  const state = previous ? structuredClone(previous) : { schemaVersion: 1, kind: 'feed-incident-state', registryId: registry.id, producerId: report.producerId, policy: p, active: {}, pending: {}, events: [], prunedEvents: 0 }
+  const transitions = []
+  const emit = (incident, transition, health, reasons) => {
+    const event = { incidentId: incident.incidentId, key: incident.key, feedId: incident.feedId, stageId: incident.stageId, firstDetectedAt: incident.firstDetectedAt, at, transition, state: health, reasons }
+    transitions.push({ id: hash(canonical(event)), ...event })
+  }
+  const observedKeys = new Set()
+  for (const condition of conditions(report, registry)) {
+    const { key, disposition, state: health, reasons, feedId, stageId } = condition
+    observedKeys.add(key)
+    let incident = state.active[key]
+    if (disposition === 'problem') {
+      if (!incident) {
+        const candidate = state.pending[key] ?? { since: at }
+        state.pending[key] = candidate
+        if (now - Date.parse(candidate.since) < p.openAfterSeconds * 1000) continue
+        incident = { key, feedId, stageId, incidentId: hash(canonical([key, candidate.since])), firstDetectedAt: candidate.since, openedAt: at, lastObservedAt: at, state: health, reasons }
+        state.active[key] = incident; delete state.pending[key]
+        emit(incident, 'opened', health, reasons)
+      } else {
+        if (incident.state !== health || canonical(incident.reasons) !== canonical(reasons)) emit(incident, 'updated', health, reasons)
+        incident.state = health; incident.reasons = reasons; incident.lastObservedAt = at
+      }
+      delete incident.recoveryStartedAt; delete incident.recoveryProducerAt
+    } else {
+      delete state.pending[key]
+      if (!incident) continue
+      if (disposition === 'hold') { delete incident.recoveryStartedAt; delete incident.recoveryProducerAt; continue }
+      incident.recoveryStartedAt ??= at
+      incident.recoveryProducerAt ??= report.producerGeneratedAt
+      // Fresh observer timestamps alone cannot make a replayed producer report confirm recovery.
+      const fresh = p.recoverAfterSeconds === 0 || Date.parse(report.producerGeneratedAt) > Date.parse(incident.recoveryProducerAt)
+      if (fresh && now - Date.parse(incident.recoveryStartedAt) >= p.recoverAfterSeconds * 1000) {
+        emit(incident, 'recovered', 'healthy', reasons)
+        delete state.active[key]
+      }
+    }
+  }
+  // Removed/paused scopes cannot silently recover existing incidents.
+  for (const key of Object.keys(state.pending)) if (!observedKeys.has(key)) delete state.pending[key]
+  for (const [key, incident] of Object.entries(state.active)) if (!observedKeys.has(key)) { delete incident.recoveryStartedAt; delete incident.recoveryProducerAt }
+  state.events.push(...transitions)
+  const cutoff = now - p.retentionDays * 86400000
+  const expired = new Set(state.events.filter(e => e.transition === 'recovered' && Date.parse(e.at) < cutoff).map(e => e.incidentId))
+  for (const incident of Object.values(state.active)) expired.delete(incident.incidentId)
+  const before = state.events.length
+  state.events = state.events.filter(e => !expired.has(e.incidentId))
+  state.prunedEvents += before - state.events.length
+  state.lastObservedAt = at; state.lastReportId = reportId; state.report = report
+  if (state.events.length > p.maxEvents) throw new Error('Incident history event budget exhausted; archive before continuing')
+  return { state: validateState(state), changed: true, events: transitions }
+}
+
+/** Single-writer transaction: checksum + fsync + atomic rename. A leftover lock fails closed. */
+export async function updateIncidentStore(root, registryValue, reportValue, { policy: requestedPolicy, now = Date.now() } = {}) {
+  const registry = readFeedRegistry(registryValue), report = readFeedHealth(reportValue, registry), p = policy(requestedPolicy)
+  if (!Number.isFinite(now) || now - Date.parse(report.observedAt) > registry.heartbeatMaxAgeSeconds * 1000 || Date.parse(report.observedAt) - now > registry.clockSkewSeconds * 1000) throw new Error('Incident observation is not current')
+  root = resolve(root)
+  // Serialize directory creation as well as updates: a second initializer must not
+  // take the writer lock before the directory creator has committed its first state.
+  const lockPath = `${root}.lock`, lock = await open(lockPath, 'wx', 0o600)
+  const temporary = join(root, `.state-${randomUUID()}`)
+  try {
+    await lock.writeFile(JSON.stringify({ pid: process.pid, host: hostname(), startedAt: new Date(now).toISOString() }))
+    await lock.sync()
+    let created = false
+    try { await mkdir(root, { mode: 0o700 }); created = true } catch (error) { if (error.code !== 'EEXIST') throw error }
+    if (!(await lstat(root)).isDirectory()) throw new Error('Incident store must be a real directory')
+    const previous = created ? null : await readIncidentStore(root)
+    const update = advance(previous, report, registry, p)
+    if (!update.changed) return update
+    const content = canonical({ sha256: hash(canonical(update.state)), state: update.state }) + '\n'
+    if (Buffer.byteLength(content) > MAX_BYTES) throw new Error('Incident history byte budget exhausted; archive before continuing')
+    const output = await open(temporary, 'wx', 0o600)
+    try { await output.writeFile(content); await output.sync() } finally { await output.close() }
+    await rename(temporary, join(root, 'state.json'))
+    const directory = await open(root, 'r')
+    try { await directory.sync() } finally { await directory.close() }
+    return update
+  } finally {
+    await lock.close()
+    try { await rm(temporary, { force: true }) } finally { await rm(lockPath) }
+  }
+}

package/feed-observability.d.mts

@@ -0,0 +1,68 @@
+export type OperationalState = 'healthy' | 'degraded' | 'unknown' | 'waiting'
+export type ConfiguredState = 'active' | 'paused' | 'setup-required'
+export type FeedSchedule = { kind: 'interval'; seconds: number } | { kind: 'daily'; timeZone: string; deadline: string } | { kind: 'manual' | 'push' }
+export interface FeedStage {
+  id: string
+  dependsOn: string[]
+  maxAgeSeconds: number | null
+  sourceMaxAgeSeconds?: number
+}
+export interface FeedDefinition {
+  id: string
+  label: string
+  producerId: string
+  producerFeed: string
+  sourceId: string
+  owner: string
+  runbook: string
+  state: ConfiguredState
+  schedule: FeedSchedule
+  stages: FeedStage[]
+}
+export interface FeedRegistry {
+  schemaVersion: 1
+  kind: 'feed-registry'
+  id: string
+  heartbeatMaxAgeSeconds: number
+  clockSkewSeconds: number
+  feeds: FeedDefinition[]
+}
+export interface FeedEvent {
+  schemaVersion: 1
+  kind: 'feed-event'
+  eventId: string
+  feedId: string
+  stageId: string
+  runId: string
+  parentRunId?: string
+  producerId: string
+  revision: string
+  occurredAt: string
+  recordedAt: string
+  type: 'started' | 'succeeded' | 'failed' | 'heartbeat'
+  reasons: string[]
+  metrics: Record<string, number>
+}
+export interface StageHealth {
+  stageId: string
+  state: OperationalState
+  reasons: string[]
+  timestamps: Partial<Record<'receivedAt' | 'sourceGeneratedAt' | 'latestPeriodEnd' | 'lastSuccessAt' | 'lastAttemptAt', string>>
+  metrics: Record<string, number>
+}
+export interface FeedHealth {
+  schemaVersion: 1
+  kind: 'feed-health'
+  registryId: string
+  producerId: string
+  observedAt: string
+  producerGeneratedAt: string | null
+  telemetry: { state: 'current' | 'stale' | 'missing' | 'invalid'; ageSeconds: number | null }
+  feeds: { feedId: string; configuredState: ConfiguredState; state: OperationalState | Exclude<ConfiguredState, 'active'>; stages: StageHealth[] }[]
+  capacity: { state: Exclude<OperationalState, 'waiting'>; reasons: string[]; metrics: Record<string, number> }
+}
+export function readOperationalTime(value: unknown): string
+export function readFeedRegistry(value: unknown): FeedRegistry
+export function readFeedEvent(value: unknown, registry: unknown): FeedEvent
+export function readFeedHealth(value: unknown, registry: unknown): FeedHealth
+export function summarizeFeedStages(stages: readonly Pick<StageHealth, 'state'>[]): OperationalState

package/feed-observability.mjs

@@ -0,0 +1,154 @@
+/** Versioned, JSON-only contracts. No I/O, clock, Node imports or provider calls. */
+const own = (o, k) => Object.hasOwn(o, k)
+function object(value, keys, label) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid ${label}`)
+  if (Object.keys(value).some(k => !keys.includes(k))) throw new Error(`Unknown ${label} field`)
+  return value
+}
+function text(value, label, limit = 160) {
+  if (typeof value !== 'string' || !value.trim() || value.length > limit || [...value].some(c => c.charCodeAt(0) < 32)) throw new Error(`Invalid ${label}`)
+  return value
+}
+function id(value) {
+  if (typeof value !== 'string' || !/^[a-z0-9][a-z0-9._:-]{0,127}$/.test(value)) throw new Error('Invalid identifier')
+  return value
+}
+function choice(value, values) {
+  if (!values.includes(value)) throw new Error('Invalid enum value')
+  return value
+}
+function number(value, max = Number.MAX_SAFE_INTEGER) {
+  if (!Number.isFinite(value) || value < 0 || value > max) throw new Error('Invalid nonnegative number')
+  return value
+}
+function seconds(value) {
+  if (!Number.isInteger(value) || value < 1 || value > 366 * 86400) throw new Error('Invalid duration')
+  return value
+}
+function array(value, max, min = 0) {
+  if (!Array.isArray(value) || value.length < min || value.length > max) throw new Error('Invalid bounded array')
+  return value
+}
+function unique(values) {
+  if (new Set(values).size !== values.length) throw new Error('Duplicate identifier')
+  return values
+}
+export function readOperationalTime(value) {
+  if (typeof value !== 'string' || !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{3})?Z$/.test(value)) throw new Error('Invalid UTC timestamp')
+  const ms = Date.parse(value)
+  if (!Number.isFinite(ms) || new Date(ms).toISOString() !== (value.length === 20 ? value.replace('Z', '.000Z') : value)) throw new Error('Invalid UTC timestamp')
+  return new Date(ms).toISOString()
+}
+function header(value, kind, keys) {
+  object(value, ['schemaVersion', 'kind', ...keys], kind)
+  if (value.schemaVersion !== 1 || value.kind !== kind) throw new Error('Unsupported operational schema')
+}
+function schedule(value) {
+  object(value, ['kind', 'seconds', 'timeZone', 'deadline'], 'schedule')
+  if (value.kind === 'interval') {
+    object(value, ['kind', 'seconds'], 'interval schedule')
+    return { kind: value.kind, seconds: seconds(value.seconds) }
+  }
+  if (value.kind === 'daily') {
+    object(value, ['kind', 'timeZone', 'deadline'], 'daily schedule')
+    text(value.timeZone, 'timezone', 80)
+    new Intl.DateTimeFormat('en', { timeZone: value.timeZone })
+    if (!/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(value.deadline)) throw new Error('Invalid daily deadline')
+    return { kind: value.kind, timeZone: value.timeZone, deadline: value.deadline }
+  }
+  object(value, ['kind'], 'manual schedule')
+  return { kind: choice(value.kind, ['manual', 'push']) }
+}
+export function readFeedRegistry(value) {
+  header(value, 'feed-registry', ['id', 'heartbeatMaxAgeSeconds', 'clockSkewSeconds', 'feeds'])
+  const feeds = array(value.feeds, 128, 1).map(f => {
+    object(f, ['id', 'label', 'producerId', 'producerFeed', 'sourceId', 'owner', 'runbook', 'state', 'schedule', 'stages'], 'feed')
+    const stages = array(f.stages, 16, 1).map(s => {
+      object(s, ['id', 'dependsOn', 'maxAgeSeconds', 'sourceMaxAgeSeconds'], 'stage')
+      return { id: id(s.id), dependsOn: unique(array(s.dependsOn, 16).map(id)),
+        maxAgeSeconds: s.maxAgeSeconds === null ? null : seconds(s.maxAgeSeconds),
+        ...(own(s, 'sourceMaxAgeSeconds') ? { sourceMaxAgeSeconds: seconds(s.sourceMaxAgeSeconds) } : {}) }
+    })
+    unique(stages.map(s => s.id))
+    const visiting = new Set(), visited = new Set()
+    const visit = stageId => {
+      if (visiting.has(stageId)) throw new Error('Cyclic stage dependency')
+      if (visited.has(stageId)) return
+      const stage = stages.find(s => s.id === stageId)
+      if (!stage) throw new Error('Unknown stage dependency')
+      visiting.add(stageId); stage.dependsOn.forEach(visit); visiting.delete(stageId); visited.add(stageId)
+    }
+    stages.forEach(s => visit(s.id))
+    return { id: id(f.id), label: text(f.label, 'label'), producerId: id(f.producerId), producerFeed: id(f.producerFeed), sourceId: id(f.sourceId),
+      owner: text(f.owner, 'owner'), runbook: text(f.runbook, 'runbook', 300), state: choice(f.state, ['active', 'paused', 'setup-required']), schedule: schedule(f.schedule), stages }
+  })
+  unique(feeds.map(f => f.id)); unique(feeds.map(f => `${f.producerId}/${f.producerFeed}`))
+  return { schemaVersion: 1, kind: 'feed-registry', id: id(value.id), heartbeatMaxAgeSeconds: seconds(value.heartbeatMaxAgeSeconds),
+    clockSkewSeconds: number(value.clockSkewSeconds, 300), feeds }
+}
+function metrics(value) {
+  object(value, Object.keys(value ?? {}), 'metrics')
+  if (Object.keys(value).length > 32) throw new Error('Too many metrics')
+  return Object.fromEntries(Object.entries(value).map(([key, v]) => [id(key), number(v)]))
+}
+function reasons(value) { return unique(array(value, 16).map(id)) }
+function registered(registry, feedId, stageId) {
+  const feed = registry.feeds.find(f => f.id === feedId)
+  if (!feed?.stages.some(s => s.id === stageId)) throw new Error('Unregistered feed or stage')
+}
+export function readFeedEvent(value, registryValue) {
+  header(value, 'feed-event', ['eventId', 'feedId', 'stageId', 'runId', 'parentRunId', 'producerId', 'revision', 'occurredAt', 'recordedAt', 'type', 'reasons', 'metrics'])
+  const registry = readFeedRegistry(registryValue)
+  registered(registry, value.feedId, value.stageId)
+  if (registry.feeds.find(f => f.id === value.feedId).producerId !== value.producerId) throw new Error('Producer does not own feed')
+  return { schemaVersion: 1, kind: 'feed-event', eventId: id(value.eventId), feedId: id(value.feedId), stageId: id(value.stageId), runId: id(value.runId),
+    ...(own(value, 'parentRunId') ? { parentRunId: id(value.parentRunId) } : {}), producerId: id(value.producerId), revision: id(value.revision),
+    occurredAt: readOperationalTime(value.occurredAt), recordedAt: readOperationalTime(value.recordedAt),
+    type: choice(value.type, ['started', 'succeeded', 'failed', 'heartbeat']), reasons: reasons(value.reasons), metrics: metrics(value.metrics) }
+}
+export function summarizeFeedStages(stages) {
+  return ['degraded', 'unknown', 'waiting'].find(state => stages.some(s => s.state === state)) ?? 'healthy'
+}
+export function readFeedHealth(value, registryValue) {
+  header(value, 'feed-health', ['registryId', 'producerId', 'observedAt', 'producerGeneratedAt', 'telemetry', 'feeds', 'capacity'])
+  const registry = readFeedRegistry(registryValue)
+  if (value.registryId !== registry.id) throw new Error('Registry does not match health report')
+  const producerId = id(value.producerId), expected = registry.feeds.filter(f => f.producerId === producerId)
+  if (!expected.length) throw new Error('Unregistered producer')
+  const observedAt = readOperationalTime(value.observedAt)
+  const producerGeneratedAt = value.producerGeneratedAt === null ? null : readOperationalTime(value.producerGeneratedAt)
+  object(value.telemetry, ['state', 'ageSeconds'], 'telemetry')
+  const telemetry = { state: choice(value.telemetry.state, ['current', 'stale', 'missing', 'invalid']), ageSeconds: value.telemetry.ageSeconds === null ? null : number(value.telemetry.ageSeconds) }
+  if (producerGeneratedAt === null) {
+    if (!['missing', 'invalid'].includes(telemetry.state) || telemetry.ageSeconds !== null) throw new Error('Missing producer clock cannot establish freshness')
+  } else {
+    const elapsed = (Date.parse(observedAt) - Date.parse(producerGeneratedAt)) / 1000
+    const state = elapsed < -registry.clockSkewSeconds ? 'invalid' : elapsed > registry.heartbeatMaxAgeSeconds ? 'stale' : 'current'
+    if (telemetry.state !== state || telemetry.ageSeconds === null || Math.abs(telemetry.ageSeconds - Math.max(0, elapsed)) > 0.000001) throw new Error('Telemetry does not match report clocks')
+  }
+  const feeds = array(value.feeds, 128, 1).map(f => {
+    object(f, ['feedId', 'configuredState', 'state', 'stages'], 'feed health')
+    const entry = expected.find(e => e.id === f.feedId)
+    if (!entry || f.configuredState !== entry.state) throw new Error('Feed configuration does not match registry')
+    const stages = array(f.stages, 16, 1).map(s => {
+      object(s, ['stageId', 'state', 'reasons', 'timestamps', 'metrics'], 'stage health')
+      registered(registry, f.feedId, s.stageId)
+      object(s.timestamps, ['receivedAt', 'sourceGeneratedAt', 'latestPeriodEnd', 'lastSuccessAt', 'lastAttemptAt'], 'timestamps')
+      return { stageId: id(s.stageId), state: choice(s.state, ['healthy', 'degraded', 'unknown', 'waiting']), reasons: reasons(s.reasons),
+        timestamps: Object.fromEntries(Object.entries(s.timestamps).map(([k, v]) => [k, readOperationalTime(v)])), metrics: metrics(s.metrics) }
+    })
+    unique(stages.map(s => s.stageId))
+    if (stages.length !== entry.stages.length) throw new Error('Incomplete stage report')
+    const state = entry.state === 'active' ? summarizeFeedStages(stages) : entry.state
+    if (f.state !== state) throw new Error('Inconsistent feed summary')
+    if (telemetry.state !== 'current' && stages.some(s => s.state !== 'unknown')) throw new Error('Unavailable telemetry cannot assert stage health')
+    return { feedId: f.feedId, configuredState: entry.state, state, stages }
+  })
+  unique(feeds.map(f => f.feedId))
+  if (feeds.length !== expected.length) throw new Error('Incomplete producer report')
+  object(value.capacity, ['state', 'reasons', 'metrics'], 'capacity')
+  const capacity = { state: choice(value.capacity.state, ['healthy', 'degraded', 'unknown']), reasons: reasons(value.capacity.reasons), metrics: metrics(value.capacity.metrics) }
+  if (telemetry.state !== 'current' && capacity.state !== 'unknown') throw new Error('Unavailable telemetry cannot assert capacity health')
+  return { schemaVersion: 1, kind: 'feed-health', registryId: registry.id, producerId, observedAt,
+    producerGeneratedAt, telemetry, feeds, capacity }
+}

package/feed-observer.d.mts

@@ -0,0 +1,30 @@
+import type { SERVICE_TIMEZONE } from './uk-service-day.mjs'
+import type { FeedHealth, StageHealth } from './feed-observability.mjs'
+import type { DailyExpectation } from './feed-expectations.mjs'
+export type ObserverCredential = string | { bearer?: string; accessClientId?: string; accessClientSecret?: string }
+export interface ObserverConfig {
+  schemaVersion: 1
+  kind: 'feed-observer-config'
+  producerId: string
+  healthUrl: string
+  consumers: { feedId: string; manifestUrl: string; operator: string; deadline: string; timeZone: typeof SERVICE_TIMEZONE }[]
+}
+export interface ObserverCheck {
+  schemaVersion: 1
+  kind: 'feed-observer-check'
+  startedAt: string
+  completedAt: string
+  sourceError: string | null
+  requests: number
+  bytes: number
+  report: FeedHealth
+  consumers: (StageHealth & { feedId: string; expectation: DailyExpectation })[]
+}
+export declare const OBSERVER_LIMITS: Readonly<{ requests: number; totalBytes: number; timeoutMs: number; totalMs: number; healthBytes: number; manifestBytes: number; releaseBytes: number }>
+export function observeFeeds(registry: unknown, config: unknown, options?: {
+  fetchImpl?: typeof fetch
+  clock?: () => number
+  tokens?: { health?: ObserverCredential; consumers?: Record<string, ObserverCredential> }
+  limits?: typeof OBSERVER_LIMITS
+}): Promise<ObserverCheck>
+export function assessObserverCheck(registry: unknown, checked: ObserverCheck | null | undefined, options?: { now?: number; maxAgeSeconds?: number }): { state: 'healthy' | 'degraded' | 'unknown'; reasons: string[] }