@motionstudies/data 0.1.0-alpha.2 → 0.1.0-alpha.21

package/source-store.mjs

@@ -0,0 +1,160 @@
+import { createHash, randomUUID } from 'node:crypto'
+import { mkdir, readFile, writeFile, rename, rm, stat, readdir, unlink } from 'node:fs/promises'
+import { join, dirname } from 'node:path'
+import { setTimeout as delay } from 'node:timers/promises'
+import { gzipSync, gunzipSync } from 'node:zlib'
+
+export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex')
+
+export function publicSourceUrl(source, value) {
+  const url = new URL(value)
+  if (url.protocol !== 'https:' || url.username || url.password || url.hash ||
+      [...url.searchParams.keys()].some(key => /key|token|password|secret|signature|credential/i.test(key))) {
+    throw new Error('Source URLs must be public HTTPS URLs without credentials or fragments')
+  }
+  if (!source.origins.includes(url.origin)) throw new Error(`URL origin is not configured for ${source.id}`)
+  return url.href
+}
+
+export async function atomicJson(path, value) {
+  await atomicWrite(path, Buffer.from(`${JSON.stringify(value, null, 2)}\n`))
+}
+
+async function atomicWrite(path, bytes) {
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 })
+  const temporary = `${path}.${randomUUID()}.partial`
+  try {
+    await writeFile(temporary, bytes, { mode: 0o600 })
+    await rename(temporary, path)
+  } finally {
+    await rm(temporary, { force: true })
+  }
+}
+
+function requestKey(source, url) { return sha256(`${source.id}\n${url}`) }
+
+export async function readCapture(store, source, url, validate = () => {}) {
+  const record = JSON.parse(await readFile(join(store, 'requests', `${requestKey(source, url)}.json`), 'utf8'))
+  if (record.source.id !== source.id || record.url !== url || !/^[a-f0-9]{64}$/.test(record.sha256)) throw new Error('Capture identity mismatch')
+  const bytes = await readObject(store, record.sha256, record.bytes)
+  if (bytes.length !== record.bytes || sha256(bytes) !== record.sha256) throw new Error('Capture integrity check failed')
+  await validate(bytes)
+  return { bytes, record, cached: true }
+}
+/** Read an object by its raw-content hash, whether it is stored raw or gzip-compressed at rest. */
+export async function readObject(store, hash, expectedBytes) {
+  try { return await readFile(join(store, 'objects', hash)) } catch (error) { if (error.code !== 'ENOENT') throw error }
+  const compressed = await readFile(join(store, 'objects', `${hash}.gz`))
+  try { return gunzipSync(compressed, { maxOutputLength: (expectedBytes ?? 512 * 1024 * 1024) + 1 }) } catch { throw new Error('Compressed capture could not be expanded within its recorded size') }
+}
+/** Store object bytes, compressing at rest when asked and worthwhile; the hash of the raw bytes remains the identity. */
+async function writeObject(store, hash, bytes, compression) {
+  if (compression === 'gzip') {
+    const compressed = gzipSync(bytes, { level: 6 })
+    if (compressed.length <= bytes.length * 0.9) { await atomicWrite(join(store, 'objects', `${hash}.gz`), compressed); return { storedBytes: compressed.length, encoding: 'gzip' } }
+  } else if (compression !== undefined) throw new Error('Unsupported compression; use gzip or leave unset')
+  await atomicWrite(join(store, 'objects', hash), bytes)
+  return { storedBytes: bytes.length, encoding: null }
+}
+/** Compress existing raw objects in a store where gzip saves at least the given fraction. Records keep their raw hash and byte count; reads follow either form. */
+export async function compactObjects(store, { minimumSaving = 0.1 } = {}) {
+  const result = { examined: 0, compressed: 0, kept: 0, rawBytes: 0, storedBytes: 0 }
+  let names
+  try { names = await readdir(join(store, 'objects')) } catch (error) { if (error.code === 'ENOENT') return result; throw error }
+  for (const name of names.filter(n => /^[a-f0-9]{64}$/.test(n))) {
+    const bytes = await readFile(join(store, 'objects', name))
+    result.examined++; result.rawBytes += bytes.length
+    const compressed = gzipSync(bytes, { level: 6 })
+    if (compressed.length <= bytes.length * (1 - minimumSaving)) {
+      await atomicWrite(join(store, 'objects', `${name}.gz`), compressed)
+      if (sha256(gunzipSync(compressed)) !== name) throw new Error('Compressed object failed verification; raw object retained')
+      await unlink(join(store, 'objects', name)); result.compressed++; result.storedBytes += compressed.length
+    } else { result.kept++; result.storedBytes += bytes.length }
+  }
+  return result
+}
+
+async function persist(store, source, url, bytes, acquisition, response = {}, compression) {
+  const hash = sha256(bytes)
+  const stored = await writeObject(store, hash, bytes, compression)
+  const record = {
+    schemaVersion: 1, source: { id: source.id, publisher: source.publisher, kind: source.kind, documentation: source.documentation, license: source.license, ...(source.attribution ? { attribution: source.attribution } : {}) },
+    url, sha256: hash, bytes: bytes.length, storedBytes: stored.storedBytes, encoding: stored.encoding,
+    ...(acquisition === 'http' ? { retrievedAt: new Date().toISOString() } : { retrievedAt: null, importedAt: new Date().toISOString() }),
+    acquisition, validation: 'raw-capture; see adapter audit for fitness', ...response,
+  }
+  await atomicJson(join(store, 'records', `${randomUUID()}.json`), record)
+  await atomicJson(join(store, 'requests', `${requestKey(source, url)}.json`), record)
+  return { bytes, record, cached: false }
+}
+
+export async function importCapture({ store, source, url, input, maxBytes = 512 * 1024 * 1024, validate = () => {}, compression }) {
+  url = publicSourceUrl(source, url)
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new Error('Expected a positive byte limit')
+  if ((await stat(input)).size > maxBytes) throw new Error('Source file exceeds byte limit')
+  const bytes = await readFile(input)
+  if (!bytes.length || bytes.length > maxBytes) throw new Error('Source file is empty or exceeds byte limit')
+  await validate(bytes)
+  return persist(store, source, url, bytes, 'local-import', {}, compression)
+}
+
+export async function capture({ store, source, url, offline = false, refresh = false, maxBytes = 32 * 1024 * 1024, timeoutMs = 20000, retries = 2, validate = () => {}, fetchImpl = fetch, userAgent = 'Motion Studies source capture', compression }) {
+  url = publicSourceUrl(source, url)
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || !Number.isSafeInteger(retries) || retries < 0 || retries > 10) throw new Error('Invalid capture limits')
+  if (offline && refresh) throw new Error('Offline and refresh cannot be combined')
+  if (!refresh) {
+    try { return await readCapture(store, source, url, validate) }
+    catch (error) { if (error.code !== 'ENOENT') throw error }
+  }
+  if (offline) throw new Error(`No cached capture for ${source.id}; offline mode never downloads`)
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    let retryable = true
+    try {
+      const response = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs), headers: { 'User-Agent': userAgent } })
+      if (response.status === 204) {
+        retryable = false
+        throw new Error('Source returned HTTP 204: no report content; this does not mean zero traffic')
+      }
+      if (!response.ok) {
+        retryable = response.status === 429 || response.status >= 500
+        await response.body?.cancel()
+        throw new Error(`Source returned HTTP ${response.status}`)
+      }
+      if (Number(response.headers.get('content-length')) > maxBytes) {
+        retryable = false
+        await response.body?.cancel()
+        throw new Error('Source exceeds byte limit')
+      }
+      if (!response.body) throw new Error('Source response has no body')
+      const reader = response.body.getReader()
+      const chunks = []
+      let size = 0
+      try {
+        while (true) {
+          const { value, done } = await reader.read()
+          if (done) break
+          size += value.length
+          if (size > maxBytes) { retryable = false; await reader.cancel(); throw new Error('Source exceeds byte limit') }
+          chunks.push(value)
+        }
+      } finally { reader.releaseLock() }
+      const bytes = Buffer.concat(chunks)
+      retryable = false
+      if (!bytes.length) throw new Error('Source response is empty')
+      await validate(bytes)
+      return await persist(store, source, url, bytes, 'http', {
+        httpStatus: response.status, contentType: response.headers.get('content-type'),
+        lastModified: response.headers.get('last-modified'), etag: response.headers.get('etag'),
+      }, compression)
+    } catch (error) {
+      if (!retryable || attempt === retries) throw error
+      await delay(250 * 2 ** attempt)
+    }
+  }
+}
+
+export function jsonDocument(bytes) {
+  const value = JSON.parse(bytes.toString('utf8'))
+  if (!value || typeof value !== 'object') throw new Error('Expected a JSON document')
+  return value
+}

package/uk-service-day.d.mts

@@ -0,0 +1,6 @@
+/** UK civil day boundaries. GTFS service time has a separate noon-based origin. */
+export interface UkStudyDay {serviceDate:string;timezone:'Europe/London';startUtc:string;endUtc:string;durationSeconds:number}
+export function checkedDate(value:string):string
+export function studyDay(serviceDate:string):UkStudyDay
+export function utcDatesForDay(day:UkStudyDay):string[]
+export function gtfsServiceInstant(serviceDate:string,seconds:number):string

package/uk-service-day.mjs

@@ -0,0 +1,49 @@
+const TIMEZONE = 'Europe/London'
+
+export function checkedDate(value) {
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(value ?? '') || !Number.isFinite(Date.parse(`${value}T00:00:00Z`)) ||
+      new Date(`${value}T00:00:00Z`).toISOString().slice(0, 10) !== value) {
+    throw new Error('An actual service date in YYYY-MM-DD form is required')
+  }
+  return value
+}
+
+// UK midnight is unambiguous on both DST transition days. Never infer a full day as 86400 seconds.
+function londonMidnight(date) {
+  const nominal = Date.parse(`${date}T00:00:00Z`)
+  const formatter = new Intl.DateTimeFormat('en-GB', { timeZone: TIMEZONE, timeZoneName: 'longOffset' })
+  let instant = nominal
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const offset = formatter.formatToParts(instant).find(part => part.type === 'timeZoneName').value
+    const match = /^GMT([+-])(\d{2}):(\d{2})$/.exec(offset)
+    if (offset !== 'GMT' && !match) throw new Error('Unsupported timezone offset')
+    const minutes = match ? (match[1] === '+' ? 1 : -1) * (Number(match[2]) * 60 + Number(match[3])) : 0
+    instant = nominal - minutes * 60000
+  }
+  return instant
+}
+
+export function studyDay(serviceDate) {
+  checkedDate(serviceDate)
+  const next = new Date(Date.parse(`${serviceDate}T00:00:00Z`) + 86400000).toISOString().slice(0, 10)
+  const start = londonMidnight(serviceDate)
+  const end = londonMidnight(next)
+  return { serviceDate, timezone: TIMEZONE, startUtc: new Date(start).toISOString(), endUtc: new Date(end).toISOString(), durationSeconds: (end - start) / 1000 }
+}
+
+export function utcDatesForDay(day) {
+  const first = Date.parse(day.startUtc.slice(0, 10))
+  const last = Date.parse(new Date(Date.parse(day.endUtc) - 1).toISOString().slice(0, 10))
+  const dates = []
+  for (let time = first; time <= last; time += 86400000) dates.push(new Date(time).toISOString().slice(0, 10))
+  return dates
+}
+
+// GTFS defines service time from local noon minus twelve elapsed hours. On UK
+// DST transition days this differs from the observed civil-day midnight.
+export function gtfsServiceInstant(serviceDate, seconds) {
+  if (!Number.isSafeInteger(seconds) || seconds < 0) throw new Error('GTFS seconds must be a nonnegative whole number')
+  const day = studyDay(serviceDate)
+  const noonMinusTwelveHours = Date.parse(day.endUtc) - 86400000
+  return new Date(noonMinusTwelveHours + seconds * 1000).toISOString()
+}

package/webtris.d.mts

@@ -0,0 +1,14 @@
+export interface WebtrisSite {id:string;description:string}
+export interface WebtrisSample {interval:number;periodEnding:string;totalVolume:number|null;averageSpeedMph:number|null;lengthClassCounts:(number|null)[];classTotalMatches:boolean|null}
+export interface WebtrisReport<S extends WebtrisSite=WebtrisSite> {
+ schemaVersion:1;kind:'observed-detector-report-audit';serviceDate:string;
+ clock:{basis:string;timezone:null;utcMapping:string;nominalIntervals:96;completeness:string};
+ lengthClasses:string[];representation:string;
+ audit:{inputRows:number;foreignSite:number;foreignDate:number;invalidInterval:number;exactDuplicates:number;conflictingIntervals:number};
+ observations:{site:S;samples:WebtrisSample[];reportIntervals:number;validVolumeIntervals:number;validSpeedIntervals:number;missingNominalIntervals:number[];conflictingIntervals:number[];classTotalMismatches:number}[];
+}
+export function reportUrl(siteIds:readonly string[],serviceDate:string,page:number,pageSize?:number):string
+/** Source slots and labels are retained. No UTC placement, flow-rate conversion or class inference. */
+export function normalizeReports<S extends WebtrisSite>(rows:readonly Record<string,unknown>[],sites:readonly S[],serviceDate:string):WebtrisReport<S>
+/** Pass stable source-record references (for example capture SHA-256 identities). */
+export function aggregateRoadSeries(observation:WebtrisReport['observations'][number],serviceDate:string,dataset:string,sourceRecordIds:readonly string[]):import('@motionstudies/core/domain/aggregate-road').AggregateRoadSeries

package/webtris.mjs

@@ -0,0 +1,67 @@
+import { checkedDate } from './uk-service-day.mjs'
+
+const ENDPOINT = 'https://webtris.nationalhighways.co.uk/api/v1.0/reports/daily'
+const CLASSES = ['0 - 520 cm', '521 - 660 cm', '661 - 1160 cm', '1160+ cm']
+const number = (v, integer = false) => {
+  if (!['number', 'string'].includes(typeof v) || String(v).trim() === '') return null
+  const n = Number(v)
+  return Number.isFinite(n) && n >= 0 && (!integer || Number.isInteger(n)) ? n : null
+}
+export function reportUrl(siteIds, serviceDate, page, pageSize = 1000) {
+  checkedDate(serviceDate)
+  if (!siteIds.length || siteIds.length > 10 || siteIds.some(id => !/^\d+$/.test(id)) || new Set(siteIds).size !== siteIds.length) throw new Error('Select 1..10 distinct numeric site IDs')
+  if (!Number.isSafeInteger(page) || page < 1 || !Number.isSafeInteger(pageSize) || pageSize < 1 || pageSize > 1000) throw new Error('Expected a positive page and page size of 1..1000')
+  const date = serviceDate.split('-').reverse().join('')
+  return `${ENDPOINT}?${new URLSearchParams({ sites: siteIds.join(','), start_date: date, end_date: date, page: String(page), page_size: String(pageSize) }).toString().replaceAll('%2C', ',')}`
+}
+
+export function normalizeReports(rows, sites, serviceDate) {
+  checkedDate(serviceDate)
+  if (!Array.isArray(rows) || !Array.isArray(sites) || sites.some(s => !s || typeof s.id !== 'string' || typeof s.description !== 'string') || new Set(sites.map(s => s.id)).size !== sites.length) throw new Error('Expected rows and unique named sites')
+  const names = new Map(sites.map(site => [site.description, site]))
+  if (names.size !== sites.length) throw new Error('Selected site descriptions are ambiguous')
+  const grouped = new Map(sites.map(site => [site.id, new Map()]))
+  const audit = { inputRows: rows.length, foreignSite: 0, foreignDate: 0, invalidInterval: 0, exactDuplicates: 0, conflictingIntervals: 0 }
+  for (const row of rows) {
+    if (!row || typeof row !== 'object' || Array.isArray(row)) throw new Error('Invalid WebTRIS row')
+    const site = names.get(row['Site Name'])
+    if (!site) { audit.foreignSite++; continue }
+    if (row['Report Date'] !== `${serviceDate}T00:00:00`) { audit.foreignDate++; continue }
+    const interval = number(row['Time Interval'], true)
+    const ending = row['Time Period Ending']
+    // Preserve the source's minute-inclusive ending label (00:14 etc). No invented UTC offset.
+    if (interval === null || interval > 99 || !/^([01]\d|2[0-3]):[0-5]\d:[0-5]\d$/.test(ending ?? '')) { audit.invalidInterval++; continue }
+    const sample = { interval, periodEnding: ending, totalVolume: number(row['Total Volume'], true), averageSpeedMph: number(row['Avg mph']), lengthClassCounts: CLASSES.map(key => number(row[key], true)) }
+    sample.classTotalMatches = sample.totalVolume === null || sample.lengthClassCounts.includes(null) ? null : sample.lengthClassCounts.reduce((a,b) => a+b,0) === sample.totalVolume
+    const map = grouped.get(site.id)
+    if (map.has(interval)) {
+      if (JSON.stringify(map.get(interval)) === JSON.stringify(sample)) audit.exactDuplicates++
+      else { if (map.get(interval) !== null) audit.conflictingIntervals++; map.set(interval, null) }
+    } else map.set(interval, sample)
+  }
+  const observations = sites.map(site => {
+    const map = grouped.get(site.id)
+    const samples = [...map.values()].filter(Boolean).sort((a,b) => a.interval-b.interval)
+    return { site, samples, reportIntervals: samples.length, validVolumeIntervals: samples.filter(x => x.totalVolume !== null).length, validSpeedIntervals: samples.filter(x => x.averageSpeedMph !== null).length,
+      missingNominalIntervals: Array.from({ length: 96 }, (_,i) => i).filter(i => !map.get(i)),
+      conflictingIntervals: [...map].filter(([,v]) => v === null).map(([i])=>i).sort((a,b)=>a-b),
+      classTotalMismatches: samples.filter(x => x.classTotalMatches === false).length }
+  })
+  return { schemaVersion: 1, kind: 'observed-detector-report-audit', serviceDate, clock: { basis: 'Provider report date, interval index and period-ending label', timezone: null, utcMapping: 'Unverified; not placed on the scene clock', nominalIntervals: 96, completeness: '96-slot reference only; DST and source time semantics require verification' }, lengthClasses: CLASSES, audit, observations,
+    representation: 'Point detector counts and reported mph; no reconstructed vehicles, light/heavy inference or gap interpolation' }
+}
+
+
+/** Translate source observations without placing their unverified clock onto UTC. */
+export function aggregateRoadSeries(observation, serviceDate, dataset, sourceRecordIds) {
+  checkedDate(serviceDate)
+  if (!dataset || !observation?.site?.id || !Array.isArray(sourceRecordIds) || !sourceRecordIds.length || sourceRecordIds.some(id => typeof id !== 'string' || !id)) throw new Error('Aggregate evidence requires dataset, site and source records')
+  return {
+    id: JSON.stringify(['webtris', dataset, observation.site.id]), provider: 'webtris', dataset, siteId: observation.site.id,
+    samples: observation.samples.map(s => ({
+      interval: { kind: 'provider-slot', serviceDate, index: s.interval, endingLabel: s.periodEnding },
+      vehicleCount: s.totalVolume, averageSpeed: s.averageSpeedMph === null ? null : { value: s.averageSpeedMph, unit: 'mph' },
+      sourceRecordIds: [...sourceRecordIds],
+    })),
+  }
+}