@motionstudies/data 0.1.0-alpha.28 → 0.1.0-alpha.29

package/README.md

@@ -200,6 +200,8 @@ The required `note` explains provenance. Optional `status`, `statusTone`, `clock
 
 `@motionstudies/data/recorder-evidence` reads explicit dated local journal/close-out and release targets, verifies their hashes and identities, and assesses missing expected outputs against deadlines. `@motionstudies/data/feed-incidents` persists debounced incident transitions with replay protection, atomic checksummed state, bounded retention and fail-closed capacity limits. These two modules require Node. See `docs/FEED-INCIDENTS.md` for CLI opt-in flags and operational limits; neither module installs a scheduler or sends notifications.
 
+`@motionstudies/data/release` checks versioned release handoffs in Node or a browser: safe file descriptors (`{ path, sha256, bytes }`), every descriptor in a manifest whatever its layout, kind and schema, and fetched bytes through Web Crypto. `@motionstudies/data/release-files` opens a release directory in Node, optionally pinned to its manifest digest, reads only described files without escaping the directory, and verifies a whole release within a byte budget. Format meaning stays with each format's reader. See the source repository's `docs/RELEASE-CONTRACT.md`.
+
 ## Optional live airport feed
 
 `AirportBoard` from `@motionstudies/web/components/AirportBoard` adds Study/Now controls around an existing `AirportHeroCard` configuration. Pass `studyCard` with the usual card props and `live={{ baseUrl, edition, airport }}` for the shared service. Import `airport-hero-card.css`. `labels` localizes the wrapper's control and availability messages. The lower-level `useAirportFeed` hook and core `domain/live-airport` contract are also public exports.

package/feed-observer.d.mts

@@ -1,6 +1,7 @@
 import type { SERVICE_TIMEZONE } from './uk-service-day.mjs'
 import type { FeedHealth, StageHealth } from './feed-observability.mjs'
 import type { DailyExpectation } from './feed-expectations.mjs'
+export type ObserverCredential = string | { bearer?: string; accessClientId?: string; accessClientSecret?: string }
 export interface ObserverConfig {
   schemaVersion: 1
   kind: 'feed-observer-config'
@@ -23,7 +24,7 @@ export declare const OBSERVER_LIMITS: Readonly<{ requests: number; totalBytes: n
 export function observeFeeds(registry: unknown, config: unknown, options?: {
   fetchImpl?: typeof fetch
   clock?: () => number
-  tokens?: { health?: string; consumers?: Record<string, string> }
+  tokens?: { health?: ObserverCredential; consumers?: Record<string, ObserverCredential> }
   limits?: typeof OBSERVER_LIMITS
 }): Promise<ObserverCheck>
 export function assessObserverCheck(registry: unknown, checked: ObserverCheck | null | undefined, options?: { now?: number; maxAgeSeconds?: number }): { state: 'healthy' | 'degraded' | 'unknown'; reasons: string[] }

package/feed-observer.mjs

@@ -35,7 +35,12 @@ function client(fetchImpl, tokens, limits) {
     const timeout = new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(new CheckError('request-timeout')) }, Math.min(limits.timeoutMs, limits.totalMs - (performance.now() - began))) })
     try {
       return await Promise.race([timeout, (async () => {
-        const headers = { Accept: 'application/json', 'Cache-Control': 'no-cache', ...(token ? { Authorization: `Bearer ${token}` } : {}) }
+        const credential = typeof token === 'string' ? { bearer: token } : token ?? {}
+        const headers = { Accept: 'application/json', 'Cache-Control': 'no-cache', ...(credential.bearer ? { Authorization: `Bearer ${credential.bearer}` } : {}) }
+        if (credential.accessClientId && credential.accessClientSecret) {
+          headers['CF-Access-Client-Id'] = credential.accessClientId
+          headers['CF-Access-Client-Secret'] = credential.accessClientSecret
+        } else if (credential.accessClientId || credential.accessClientSecret) throw new CheckError('credentials-invalid')
         const response = await fetchImpl(url, { headers, signal: controller.signal, redirect: 'error', cache: 'no-store' })
         body = response.body
         if (!response.ok || response.redirected) throw new CheckError('http-unavailable')

package/operational-files.mjs

@@ -6,7 +6,8 @@ export async function readOperationalFile(path, limit) {
   try {
     const stat = await handle.stat()
     if (!stat.isFile() || stat.size > limit) throw new Error('Input must be a bounded regular file')
-    const bytes = Buffer.alloc(limit + 1)
+    // Size the buffer from the file, not the limit, so a generous limit costs nothing for small inputs; one spare byte detects growth.
+    const bytes = Buffer.alloc(Math.min(limit, stat.size) + 1)
     let size = 0
     while (size <= limit) {
       const { bytesRead } = await handle.read(bytes, size, bytes.length - size, null)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@motionstudies/data",
-  "version": "0.1.0-alpha.28",
+  "version": "0.1.0-alpha.29",
   "private": false,
   "type": "module",
   "description": "Offline data preparation primitives for Motion Studies.",
@@ -59,6 +59,14 @@
       "types": "./source-store.d.mts",
       "import": "./source-store.mjs"
     },
+    "./release": {
+      "types": "./release.d.mts",
+      "import": "./release.mjs"
+    },
+    "./release-files": {
+      "types": "./release-files.d.mts",
+      "import": "./release-files.mjs"
+    },
     "./uk-service-day": {
       "types": "./uk-service-day.d.mts",
       "import": "./uk-service-day.mjs"
@@ -125,7 +133,7 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@motionstudies/core": "0.1.0-alpha.28",
+    "@motionstudies/core": "0.1.0-alpha.29",
     "fast-xml-parser": "5.11.1"
   }
 }

package/recorder-evidence.d.mts

@@ -10,5 +10,5 @@ export interface RecorderEvidencePlan {
     publication?: { pointerPath: string; operator: string; expectedThrough: string; deadlineAt: string }
   }[]
 }
-/** Reads explicitly selected local files, with integrity checks and a 64 MiB total read budget. */
+/** Reads explicitly selected local files, with integrity checks, 256 MiB per journal and a 512 MiB total read budget. */
 export function recorderEvidenceHealth(registry: unknown, report: unknown, plan: unknown): Promise<FeedHealth>

package/recorder-evidence.mjs

@@ -47,7 +47,8 @@ const pending = (id, deadline, now, reason) => result(id, now > Date.parse(deadl
 export async function recorderEvidenceHealth(registryValue, reportValue, planValue) {
   const registry = readFeedRegistry(registryValue), report = readFeedHealth(reportValue, registry)
   const plan = readPlan(planValue, registry, report.producerId), now = Date.parse(report.observedAt)
-  let remaining = 64 * 1024 * 1024
+  // A national bus day's journal is about 140 MiB; smaller feeds are a few MiB.
+  let remaining = 512 * 1024 * 1024
   const read = async (path, limit) => {
     if (remaining <= 0) throw new Error('Evidence byte budget exceeded')
     const bytes = await readOperationalFile(path, Math.min(limit, remaining))
@@ -74,7 +75,7 @@ export async function recorderEvidenceHealth(registryValue, reportValue, planVal
 }
 
 async function archive(plan, entry, read, now, skew) {
-  const { bytes, json: journal } = await read(plan.journalPath, 16 * 1024 * 1024)
+  const { bytes, json: journal } = await read(plan.journalPath, 256 * 1024 * 1024)
   if (!journal.id || journal.config?.source !== entry.sourceId || !['complete', 'stopped'].includes(journal.status) || !Array.isArray(journal.attempts)) throw new Error('Invalid closed journal')
   const end = readOperationalTime(journal.plannedEnd), start = readOperationalTime(journal.startedAt)
   if (end !== plan.expectedEnd || Date.parse(end) < Date.parse(start) || Date.parse(end) > now || Date.parse(plan.deadlineAt) < Date.parse(end)) throw new Error('Invalid archive period/deadline')

package/release-files.d.mts

@@ -0,0 +1,14 @@
+import type { ReleaseDescriptor } from './release.mjs'
+export interface ReleaseOpenOptions { kind?: string; schemaVersions?: readonly number[]; manifestSha256?: string; maxManifestBytes?: number; manifest?: string }
+export interface OpenedRelease<M = Record<string, unknown>> {
+  manifest: M
+  manifestBytes: Buffer
+  descriptors: ReleaseDescriptor[]
+  /** Read a file the manifest describes, by path or descriptor, checking size and digest. */
+  read(file: string | ReleaseDescriptor, options?: { maxBytes?: number }): Promise<Buffer>
+}
+/** Bounded read without following a final symlink or blocking on a FIFO. */
+export function readOperationalFile(path: string, limit: number): Promise<Buffer>
+export function readReleaseFile(root: string, descriptor: ReleaseDescriptor, options?: { maxBytes?: number }): Promise<Buffer>
+export function openRelease<M = Record<string, unknown>>(root: string, options?: ReleaseOpenOptions): Promise<OpenedRelease<M>>
+export function verifyRelease<M = Record<string, unknown>>(root: string, options?: ReleaseOpenOptions & { maxTotalBytes?: number }): Promise<{ manifest: M; manifestSha256: string; files: number; bytes: number }>

package/release-files.mjs

@@ -0,0 +1,47 @@
+import { realpath } from 'node:fs/promises'
+import { join, relative, isAbsolute, resolve, sep } from 'node:path'
+import { readOperationalFile } from './operational-files.mjs'
+import { checkReleaseManifest, digestHex, isReleasePath, readDescriptor, releaseDescriptors, verifyDescriptorBytes } from './release.mjs'
+
+export { readOperationalFile }
+const inside = (root, path) => { const r = relative(root, path); return r !== '' && !r.startsWith(`..${sep}`) && r !== '..' && !isAbsolute(r) }
+
+/** Read one described file from a release directory: safe path, no escape through links, bounded, size and digest checked. */
+export async function readReleaseFile(root, descriptor, { maxBytes = 512 * 1024 ** 2 } = {}) {
+  const d = readDescriptor(descriptor)
+  if (d.bytes > maxBytes) throw new Error(`${d.path} exceeds the read limit`)
+  const base = await realpath(resolve(root)), actual = await realpath(join(base, ...d.path.split('/')))
+  if (!inside(base, actual)) throw new Error(`${d.path} resolves outside the release`)
+  return verifyDescriptorBytes(await readOperationalFile(actual, d.bytes), d)
+}
+
+/**
+ * Open a release directory: read `manifest.json` (bounded), optionally require its pinned digest, check its kind and
+ * schema, and enumerate its files. Nothing but the manifest is read until asked.
+ */
+export async function openRelease(root, { kind, schemaVersions, manifestSha256, maxManifestBytes = 8 * 1024 ** 2, manifest: name = 'manifest.json' } = {}) {
+  if (!isReleasePath(name)) throw new Error('Unsafe manifest path')
+  const bytes = await readOperationalFile(join(resolve(root), name), maxManifestBytes)
+  if (manifestSha256 !== undefined && await digestHex(bytes) !== manifestSha256) throw new Error('Release manifest does not match its pinned digest')
+  const manifest = checkReleaseManifest(JSON.parse(bytes.toString('utf8')), { kind, schemaVersions })
+  const descriptors = releaseDescriptors(manifest), byPath = new Map(descriptors.map(d => [d.path, d]))
+  return {
+    manifest, manifestBytes: bytes, descriptors,
+    /** Read a described file by path or descriptor; a path the manifest does not list is refused. */
+    read: async (file, options) => {
+      const d = typeof file === 'string' ? byPath.get(file) : file
+      const listed = d && byPath.get(d.path)
+      if (!listed || listed.sha256 !== d.sha256 || listed.bytes !== d.bytes) throw new Error(`${typeof file === 'string' ? file : file?.path} is not described by the manifest`)
+      return readReleaseFile(root, d, options)
+    },
+  }
+}
+
+/** Verify every file a release describes against a total byte budget. */
+export async function verifyRelease(root, { maxTotalBytes = 2 * 1024 ** 3, ...options } = {}) {
+  const release = await openRelease(root, options)
+  const total = release.descriptors.reduce((n, d) => n + d.bytes, 0)
+  if (total > maxTotalBytes) throw new Error('Release exceeds the verification budget')
+  for (const d of release.descriptors) await readReleaseFile(root, d)
+  return { manifest: release.manifest, manifestSha256: await digestHex(release.manifestBytes), files: release.descriptors.length, bytes: total }
+}

package/release.d.mts

@@ -0,0 +1,10 @@
+/** A file a release describes: its release-relative path and the size and SHA-256 of its stored bytes. */
+export interface ReleaseDescriptor { path: string; sha256: string; bytes: number; [field: string]: unknown }
+export declare const RELEASE_DIGEST: RegExp
+export function isReleasePath(path: unknown): path is string
+export function readDescriptor(value: unknown, label?: string): ReleaseDescriptor
+/** Every descriptor in a manifest's `files` (array or map), `objects`, `chunks`, `list` and named top-level descriptors, sorted by path. */
+export function releaseDescriptors(manifest: unknown): ReleaseDescriptor[]
+export function checkReleaseManifest<T extends object>(manifest: T, options: { kind?: string; schemaVersions?: readonly number[] }): T
+export function digestHex(bytes: ArrayBuffer | ArrayBufferView): Promise<string>
+export function verifyDescriptorBytes<T extends ArrayBuffer | ArrayBufferView>(bytes: T, descriptor: ReleaseDescriptor): Promise<T>

package/release.mjs

@@ -0,0 +1,62 @@
+// Shared checks for versioned release handoffs. Runs in Node and in browsers (Web Crypto only).
+
+export const RELEASE_DIGEST = /^[a-f0-9]{64}$/
+const SEGMENT = /^[A-Za-z0-9._-]{1,255}$/
+
+/** A release-relative path: forward-slash segments of safe characters, never absolute, empty, `.` or `..`. */
+export function isReleasePath(path) {
+  return typeof path === 'string' && path.length <= 1024 && path.split('/').every(s => SEGMENT.test(s) && s !== '.' && s !== '..')
+}
+
+/** Validate one `{ path, sha256, bytes }` file descriptor; extra fields such as `decodedBytes` are kept. */
+export function readDescriptor(value, label = 'file') {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid ${label} descriptor`)
+  if (!isReleasePath(value.path)) throw new Error(`Unsafe ${label} path`)
+  if (!RELEASE_DIGEST.test(value.sha256 ?? '')) throw new Error(`Invalid ${label} digest`)
+  if (!Number.isSafeInteger(value.bytes) || value.bytes < 0) throw new Error(`Invalid ${label} size`)
+  return value
+}
+
+const looksLikeDescriptor = v => v && typeof v === 'object' && !Array.isArray(v) && typeof v.path === 'string' && 'sha256' in v && 'bytes' in v
+/**
+ * Every file a manifest describes, whatever its layout: a `files` array or name-keyed map, an `objects` map keyed by
+ * digest, `chunks`, a published bus day's `list`, and named top-level descriptors such as `day` or `index`. The same
+ * path may appear more than once (air releases list chunks in `files` and `chunks`) but must then describe the same
+ * bytes.
+ */
+export function releaseDescriptors(manifest) {
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) throw new Error('Invalid release manifest')
+  const found = new Map()
+  const add = (value, label) => {
+    const d = readDescriptor(value, label), seen = found.get(d.path)
+    if (seen && (seen.sha256 !== d.sha256 || seen.bytes !== d.bytes)) throw new Error(`Conflicting descriptors for ${d.path}`)
+    if (!seen) found.set(d.path, d)
+  }
+  for (const [key, value] of Object.entries(manifest)) {
+    if (Array.isArray(value) && ['files', 'chunks', 'list'].includes(key)) value.forEach((v, i) => add(v, `${key}[${i}]`))
+    else if (['files', 'objects'].includes(key) && value && typeof value === 'object' && !Array.isArray(value)) for (const [name, v] of Object.entries(value)) add(v, `${key}.${name}`)
+    else if (looksLikeDescriptor(value)) add(value, key)
+  }
+  return [...found.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
+}
+
+/** Check a manifest's identity before trusting its layout. */
+export function checkReleaseManifest(manifest, { kind, schemaVersions }) {
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) throw new Error('Invalid release manifest')
+  if (kind !== undefined && manifest.kind !== kind) throw new Error(`Expected a ${kind} release`)
+  if (schemaVersions !== undefined && !schemaVersions.includes(manifest.schemaVersion)) throw new Error(`Unsupported ${manifest.kind ?? 'release'} schema version`)
+  return manifest
+}
+
+const hex = buffer => Array.from(new Uint8Array(buffer), b => b.toString(16).padStart(2, '0')).join('')
+/** SHA-256 of bytes as lowercase hex, through Web Crypto. */
+export async function digestHex(bytes) {
+  return hex(await globalThis.crypto.subtle.digest('SHA-256', bytes))
+}
+/** Throw unless `bytes` are exactly the descriptor's size and digest. */
+export async function verifyDescriptorBytes(bytes, descriptor) {
+  const d = readDescriptor(descriptor)
+  if (bytes.byteLength !== d.bytes) throw new Error(`${d.path} has ${bytes.byteLength} bytes, expected ${d.bytes}`)
+  if (await digestHex(bytes) !== d.sha256) throw new Error(`${d.path} does not match its digest`)
+  return bytes
+}