@motionstudies/data 0.1.0-alpha.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Motion Studies contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,26 @@
+# Motion Studies packages
+
+Shared packages for the Motion Studies transport instrument. The source workspace and compiled distributions expose the same explicit, extensionless module subpaths. The initial release is `0.1.0-alpha.0` under npm’s `next` tag.
+
+- `@motionstudies/core`: transport contracts, indexing, interpolation and visual theme contracts; no browser or Node dependencies.
+- `@motionstudies/three`: `NationalNetworkScene`, `HubPulseScene`, `StationFlowScene`, camera framing and label-mode contracts. React, React Three Fiber and Three.js are peers; rendering internals are not public subpaths.
+- `@motionstudies/web`: picker, theme application, mounting, progressive loaders, observed operations and recording. Import `tokens.css` and `mobile-picker.css` for isolated widgets. `shell.css` is an optional full-page study shell scoped to `.motion-study`; `mountMotionStudy` applies that class. Fonts and edition layouts belong to consumers.
+- `@motionstudies/data`: Node-only GTFS readers, network chunking, merging and station ranking. ZIP reading requires `unzip` on the host. Source selection, provenance overrides and compilation commands belong to each edition.
+
+```tsx
+import { MobilePicker } from '@motionstudies/web/components/MobilePicker'
+import { createDataUrlResolver } from '@motionstudies/web/data-url'
+import { useProgressiveNetworkDay } from '@motionstudies/web/use-progressive-network-day'
+import '@motionstudies/web/tokens.css'
+import '@motionstudies/web/mobile-picker.css'
+
+const resolveData = createDataUrlResolver('/my-edition/data/')
+// Inside a React component:
+// const day = useProgressiveNetworkDay('day.json', enabled, time, resolveData)
+```
+
+Keep the resolver stable across renders. Manifest paths and their chunk paths are relative to the supplied asset root. Changing the resolved manifest URL resets the loader; disabling cancels requests, and re-enabling retries failures. Optional road manifests distinguish a 404 from an error. Observations poll serially; changing their endpoint drops the previous source's state, and disabling/unmounting cancels the request and timer.
+
+Build release candidates with `npm run build:packages`. Distribution manifests and compiled ESM/declarations are written to `.package-dist/`; workspace manifests continue to point at source for fast local iteration. `npm run check:packed` packs and installs those distributions into a separate consumer, builds the lab and validates the public exports. No source aliases or workspace links are used in that consumer.
+
+Source workspace manifests always stay private. `npm run check:release` builds public candidates, tests their packed consumer and records the tested tarball hashes; `npm run release:dry-run` inspects the publication without writing to npm. The manual main-branch `release.yml` workflow publishes those same tarballs with public access and provenance. See [release instructions](https://github.com/emmettl/motionstudies/blob/main/docs/RELEASING.md) for bootstrap-token and trusted-publisher setup. All four shared packages are MIT-licensed; each distribution includes `LICENSE`.

package/gtfs.d.mts

@@ -0,0 +1,11 @@
+export type GtfsRow = Record<string, string>
+export type TransportMode = 'tram' | 'metro' | 'rail' | 'bus' | 'ferry' | 'cableway' | 'funicular'
+export interface SourceStop { name: string; latitude: number; longitude: number; platformCode: string }
+export function parseCsvLine(line: string): string[]
+export function parseGtfsTime(value: string): number
+export function weekdayField(date: string): string
+export function transportModeForRouteType(value: string | number): TransportMode | undefined
+/** Streams a ZIP entry using the host's unzip executable. */
+export function rowsFromArchive(archive: string, fileName: string): AsyncGenerator<GtfsRow>
+export function activeServices(archive: string, serviceDate: string): Promise<Set<string>>
+export function stopsById(archive: string): Promise<Map<string, SourceStop>>

package/gtfs.mjs

@@ -0,0 +1,148 @@
+import { spawn } from 'node:child_process'
+import { createInterface } from 'node:readline'
+
+export function parseCsvLine(line) {
+  const values = []
+  let value = ''
+  let quoted = false
+
+  for (let index = 0; index < line.length; index += 1) {
+    const character = line[index]
+    if (character === '"') {
+      if (quoted && line[index + 1] === '"') {
+        value += '"'
+        index += 1
+      } else {
+        quoted = !quoted
+      }
+    } else if (character === ',' && !quoted) {
+      values.push(value)
+      value = ''
+    } else {
+      value += character
+    }
+  }
+
+  values.push(value)
+  return values
+}
+
+export function parseGtfsTime(value) {
+  const [hours, minutes, seconds] = value.split(':').map(Number)
+  return hours * 3600 + minutes * 60 + seconds
+}
+
+function compactDate(value) {
+  return value.replaceAll('-', '')
+}
+
+export function weekdayField(date) {
+  return [
+    'sunday',
+    'monday',
+    'tuesday',
+    'wednesday',
+    'thursday',
+    'friday',
+    'saturday',
+  ][new Date(`${date}T12:00:00Z`).getUTCDay()]
+}
+
+export function transportModeForRouteType(value) {
+  const routeType = Number(value)
+  if (routeType === 0 || (routeType >= 900 && routeType < 1000)) return 'tram'
+  if (routeType === 1 || (routeType >= 400 && routeType < 500)) return 'metro'
+  if (routeType === 2 || (routeType >= 100 && routeType < 200)) return 'rail'
+  if (
+    routeType === 3 ||
+    routeType === 11 ||
+    (routeType >= 200 && routeType < 300) ||
+    (routeType >= 700 && routeType < 800)
+  ) return 'bus'
+  if (routeType === 4 || (routeType >= 1000 && routeType < 1100)) return 'ferry'
+  if (routeType === 6 || (routeType >= 1300 && routeType < 1400)) {
+    return 'cableway'
+  }
+  if (routeType === 7 || (routeType >= 1400 && routeType < 1500)) {
+    return 'funicular'
+  }
+  return undefined
+}
+
+export async function* rowsFromArchive(archive, fileName) {
+  const child = spawn('unzip', ['-p', archive, fileName], {
+    stdio: ['ignore', 'pipe', 'pipe'],
+  })
+  let standardError = ''
+  child.stderr.setEncoding('utf8')
+  child.stderr.on('data', (chunk) => {
+    standardError += chunk
+  })
+  const completed = new Promise((resolvePromise, rejectPromise) => {
+    child.once('error', rejectPromise)
+    child.once('close', (code) => {
+      if (code === 0) resolvePromise()
+      else rejectPromise(new Error(`unzip ${fileName} failed: ${standardError.trim()}`))
+    })
+  })
+
+  // Observe rejection immediately; the generator may still be consuming stdout.
+  void completed.catch(() => {})
+  const lines = createInterface({ input: child.stdout, crlfDelay: Infinity })
+  try {
+  let headers
+  for await (const line of lines) {
+    if (!headers) {
+      headers = parseCsvLine(line.replace(/^\uFEFF/, ''))
+      continue
+    }
+    const values = parseCsvLine(line)
+    const row = Object.create(null)
+    for (let index = 0; index < headers.length; index += 1) {
+      row[headers[index]] = values[index] ?? ''
+    }
+    yield row
+  }
+  await completed
+  } finally {
+    lines.close()
+    if (child.exitCode === null) child.kill()
+    await completed.catch(() => {})
+  }
+}
+
+export async function activeServices(archive, serviceDate) {
+  const date = compactDate(serviceDate)
+  const weekday = weekdayField(serviceDate)
+  const services = new Set()
+
+  for await (const row of rowsFromArchive(archive, 'calendar.txt')) {
+    if (row.start_date <= date && row.end_date >= date && row[weekday] === '1') {
+      services.add(row.service_id)
+    }
+  }
+
+  for await (const row of rowsFromArchive(archive, 'calendar_dates.txt')) {
+    if (row.date !== date) continue
+    if (row.exception_type === '1') services.add(row.service_id)
+    if (row.exception_type === '2') services.delete(row.service_id)
+  }
+
+  return services
+}
+
+export async function stopsById(archive) {
+  const stops = new Map()
+  for await (const row of rowsFromArchive(archive, 'stops.txt')) {
+    const latitude = Number(row.stop_lat)
+    const longitude = Number(row.stop_lon)
+    if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) continue
+    stops.set(row.stop_id, {
+      name: row.stop_name,
+      latitude,
+      longitude,
+      platformCode: row.platform_code,
+    })
+  }
+  return stops
+}

package/merge-network.d.mts

@@ -0,0 +1,8 @@
+import type { NetworkSnapshot } from '@motionstudies/core/domain/network'
+export interface MergeOptions {
+  retrievedAt?: string
+  note?: string
+  metadata?: Partial<NetworkSnapshot['metadata']>
+  geometryMetadata?: Partial<NonNullable<NetworkSnapshot['metadata']['geometry']>>
+}
+export function mergeNetworkSnapshots(snapshots: readonly NetworkSnapshot[], options?: MergeOptions): NetworkSnapshot

package/merge-network.mjs

@@ -0,0 +1,118 @@
+import { createHash } from 'node:crypto'
+
+function boundsFor(stops) {
+  return stops.reduce((bounds, [longitude, latitude]) => ({
+    minLongitude: Math.min(bounds.minLongitude, longitude),
+    minLatitude: Math.min(bounds.minLatitude, latitude),
+    maxLongitude: Math.max(bounds.maxLongitude, longitude),
+    maxLatitude: Math.max(bounds.maxLatitude, latitude),
+  }), {
+    minLongitude: Number.POSITIVE_INFINITY,
+    minLatitude: Number.POSITIVE_INFINITY,
+    maxLongitude: Number.NEGATIVE_INFINITY,
+    maxLatitude: Number.NEGATIVE_INFINITY,
+  })
+}
+
+function sha256(value) {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex')
+}
+
+export function mergeNetworkSnapshots(snapshots, {
+  retrievedAt = new Date().toISOString(),
+  note,
+  metadata = {},
+  geometryMetadata = {},
+} = {}) {
+  if (!snapshots.length) throw new Error('At least one network snapshot is required')
+  const first = snapshots[0]
+  for (const snapshot of snapshots.slice(1)) {
+    if (snapshot.metadata.serviceDate !== first.metadata.serviceDate ||
+      snapshot.metadata.windowStart !== first.metadata.windowStart ||
+      snapshot.metadata.windowEnd !== first.metadata.windowEnd) {
+      throw new Error('Network snapshots must share a service date and study window')
+    }
+  }
+
+  const stops = []
+  const stopIndexBySourceId = new Map()
+  const paths = []
+  const pathIndexByCoordinates = new Map()
+  const edges = []
+  const edgePaths = []
+  const trains = []
+
+  for (const snapshot of snapshots) {
+    const stopRemap = snapshot.stops.map((stop) => {
+      const sourceId = stop[4] ?? `${stop[0]}:${stop[1]}:${stop[2]}`
+      const existing = stopIndexBySourceId.get(sourceId)
+      if (existing !== undefined) return existing
+      const index = stops.length
+      stops.push(stop)
+      stopIndexBySourceId.set(sourceId, index)
+      return index
+    })
+    const pathRemap = (snapshot.paths ?? []).map((path) => {
+      const key = JSON.stringify(path)
+      const existing = pathIndexByCoordinates.get(key)
+      if (existing !== undefined) return existing
+      const index = paths.length
+      paths.push(path)
+      pathIndexByCoordinates.set(key, index)
+      return index
+    })
+    snapshot.edges.forEach(([from, to], index) => {
+      edges.push([stopRemap[from], stopRemap[to]])
+      const sourcePath = snapshot.edgePaths ? snapshot.edgePaths[index] : index
+      edgePaths.push(sourcePath == null ? null : pathRemap[sourcePath] ?? null)
+    })
+    trains.push(...snapshot.trains.map((train) => ({
+      ...train,
+      stops: train.stops.map(([stopIndex, arrival, departure]) => [stopRemap[stopIndex], arrival, departure]),
+      pathSegments: train.pathSegments?.map((pathIndex) => pathIndex === null ? null : pathRemap[pathIndex] ?? null),
+    })))
+  }
+
+  const sources = snapshots.map(({ metadata }) => ({
+    modes: metadata.modes,
+    sourceUrl: metadata.sourceUrl,
+    sourceSha256: metadata.sourceSha256,
+    geometrySourceUrl: metadata.geometry?.sourceUrl,
+    geometrySourceSha256: metadata.geometry?.sourceSha256,
+    model: metadata.model,
+    note: metadata.note,
+  }))
+  const modes = [...new Set(snapshots.flatMap(({ metadata }) => metadata.modes ?? []))]
+  return {
+    metadata: {
+      ...first.metadata,
+      ...metadata,
+      serviceDate: first.metadata.serviceDate,
+      windowStart: first.metadata.windowStart,
+      windowEnd: first.metadata.windowEnd,
+      focusTime: first.metadata.focusTime,
+      sourceSha256: sha256(sources),
+      retrievedAt,
+      note: note ?? `${snapshots.length} studies merged into one network.`,
+      modes,
+      sources,
+      geometry: {
+        publisher: first.metadata.publisher,
+        feedVersion: first.metadata.feedVersion,
+        sourceUrl: first.metadata.sourceUrl,
+        model: 'Deduplicated paths from component networks',
+        ...first.metadata.geometry,
+        ...geometryMetadata,
+        sourceSha256: sha256(sources.map(({ geometrySourceSha256 }) => geometrySourceSha256)),
+        matchedSegments: paths.length,
+        totalSegments: paths.length,
+      },
+    },
+    bounds: boundsFor(stops),
+    stops,
+    edges,
+    paths,
+    edgePaths,
+    trains,
+  }
+}

package/network-chunks.d.mts

@@ -0,0 +1,6 @@
+import type { NetworkSnapshot, NetworkDayManifest, NetworkDayChunkDescriptor, NetworkDayChunk } from '@motionstudies/core/domain/network'
+export function extractNetworkWindow(snapshot: NetworkSnapshot, windowStart: number, windowEnd: number, focusTime: number): NetworkSnapshot
+export function chunkNetworkSnapshot(snapshot: NetworkSnapshot, chunkSeconds: number, chunkDirectoryName: string): {
+  manifest: NetworkDayManifest
+  chunks: { descriptor: NetworkDayChunkDescriptor; payload: NetworkDayChunk }[]
+}

package/network-chunks.mjs

@@ -0,0 +1,71 @@
+import { createHash } from 'node:crypto'
+
+export function extractNetworkWindow(snapshot, windowStart, windowEnd, focusTime) {
+  if (windowStart >= windowEnd) throw new Error('Network window must have positive duration')
+  return {
+    ...snapshot,
+    metadata: {
+      ...snapshot.metadata,
+      windowStart,
+      windowEnd,
+      focusTime,
+    },
+    trains: snapshot.trains.filter(
+      (train) => train.start <= windowEnd && train.end >= windowStart,
+    ),
+  }
+}
+
+export function chunkNetworkSnapshot(snapshot, chunkSeconds, chunkDirectoryName) {
+  if (!Number.isSafeInteger(chunkSeconds) || chunkSeconds <= 0) {
+    throw new Error('Chunk duration must be a positive whole number of seconds')
+  }
+  if (!Number.isSafeInteger(snapshot.metadata.windowStart) || !Number.isSafeInteger(snapshot.metadata.windowEnd) || snapshot.metadata.windowEnd <= snapshot.metadata.windowStart) {
+    throw new Error('Network window must have positive duration in whole seconds')
+  }
+  const clockId = (time) => `${String(Math.floor(time / 3600)).padStart(2, '0')}h${String(Math.floor(time / 60) % 60).padStart(2, '0')}m${String(time % 60).padStart(2, '0')}s`
+  const chunks = []
+  for (
+    let windowStart = snapshot.metadata.windowStart;
+    windowStart < snapshot.metadata.windowEnd;
+    windowStart += chunkSeconds
+  ) {
+    const windowEnd = Math.min(snapshot.metadata.windowEnd, windowStart + chunkSeconds)
+    const startHour = String(Math.floor(windowStart / 3600)).padStart(2, '0')
+    const endHour = String(Math.ceil(windowEnd / 3600)).padStart(2, '0')
+    const id = windowStart % 3600 === 0 && windowEnd % 3600 === 0
+      ? `${startHour}-${endHour}`
+      : `${clockId(windowStart)}-${clockId(windowEnd)}`
+    const trains = snapshot.trains.filter(
+      (train) => train.start <= windowEnd && train.end >= windowStart,
+    )
+    const payload = { windowStart, windowEnd, trains }
+    const encoded = JSON.stringify(payload)
+    chunks.push({
+      descriptor: {
+        id,
+        windowStart,
+        windowEnd,
+        path: `${chunkDirectoryName}/${id}.json`,
+        tripCount: trains.length,
+        bytes: Buffer.byteLength(encoded),
+        sha256: createHash('sha256').update(encoded).digest('hex'),
+      },
+      payload,
+    })
+  }
+
+  return {
+    manifest: {
+      metadata: snapshot.metadata,
+      bounds: snapshot.bounds,
+      stops: snapshot.stops,
+      edges: snapshot.edges,
+      ...(snapshot.paths ? { paths: snapshot.paths } : {}),
+      ...(snapshot.edgePaths ? { edgePaths: snapshot.edgePaths } : {}),
+      tripCount: snapshot.trains.length,
+      chunks: chunks.map(({ descriptor }) => descriptor),
+    },
+    chunks,
+  }
+}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@motionstudies/data",
+  "version": "0.1.0-alpha.0",
+  "private": false,
+  "type": "module",
+  "description": "Offline data preparation primitives for Motion Studies.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/emmettl/motionstudies.git",
+    "directory": "packages/data"
+  },
+  "homepage": "https://github.com/emmettl/motionstudies#readme",
+  "bugs": {
+    "url": "https://github.com/emmettl/motionstudies/issues"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/",
+    "tag": "next"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "exports": {
+    "./gtfs": {
+      "types": "./gtfs.d.mts",
+      "import": "./gtfs.mjs"
+    },
+    "./network-chunks": {
+      "types": "./network-chunks.d.mts",
+      "import": "./network-chunks.mjs"
+    },
+    "./station-ranking": {
+      "types": "./station-ranking.d.mts",
+      "import": "./station-ranking.mjs"
+    },
+    "./merge-network": {
+      "types": "./merge-network.d.mts",
+      "import": "./merge-network.mjs"
+    }
+  },
+  "files": [
+    "**/*.js",
+    "**/*.mjs",
+    "**/*.d.ts",
+    "**/*.d.mts",
+    "**/*.css",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "dependencies": {
+    "@motionstudies/core": "0.1.0-alpha.0"
+  }
+}

package/station-ranking.d.mts

@@ -0,0 +1,19 @@
+import type { NetworkSnapshot } from '@motionstudies/core/domain/network'
+export interface StationRank {
+  name: string
+  labelRank: number
+  modeCount: number
+  routeCount: number
+  movementCount: number
+  neighbourCount: number
+  score: number
+}
+export interface RankingOptions {
+  catalogue?: { lines?: readonly {
+    mode: string
+    name: string
+    directions?: readonly { branches: readonly { stopIds?: readonly string[] }[] }[]
+  }[] }
+}
+export function rankNetworkStations(snapshot: NetworkSnapshot, options?: RankingOptions): StationRank[]
+export function applyStationLabelRanks(snapshot: NetworkSnapshot, options?: RankingOptions): NetworkSnapshot

package/station-ranking.mjs

@@ -0,0 +1,101 @@
+
+
+export function rankNetworkStations(snapshot, { catalogue } = {}) {
+  const records = new Map()
+  const nameByStopIndex = snapshot.stops.map((stop, stopIndex) => {
+    const name = stop[2]
+    const record = records.get(name) ?? {
+      name,
+      stopIndexes: [],
+      trainIds: new Set(),
+      routes: new Set(),
+      modes: new Set(),
+      neighbours: new Set(),
+    }
+    record.stopIndexes.push(stopIndex)
+    records.set(name, record)
+    return name
+  })
+
+  for (const train of snapshot.trains) {
+    const visited = new Set()
+    for (const [stopIndex] of train.stops) {
+      const name = nameByStopIndex[stopIndex]
+      if (!name || visited.has(name)) continue
+      visited.add(name)
+      const record = records.get(name)
+      record.trainIds.add(train.id)
+      record.routes.add(`${train.mode ?? train.category}:${train.route}`)
+      record.modes.add(train.mode ?? train.category)
+    }
+  }
+
+  if (catalogue) {
+    const nameBySourceId = new Map(snapshot.stops.map((stop) => [stop[4], stop[2]]))
+    for (const line of catalogue.lines ?? []) {
+      const visitedNames = new Set()
+      for (const branch of line.directions?.flatMap(({ branches }) => branches) ?? []) {
+        for (const sourceId of branch.stopIds ?? []) {
+          const name = nameBySourceId.get(sourceId)
+          if (!name || visitedNames.has(name)) continue
+          visitedNames.add(name)
+          const record = records.get(name)
+          record.routes.add(`${line.mode}:${line.name}`)
+          record.modes.add(line.mode)
+        }
+      }
+    }
+  }
+
+  for (const [fromIndex, toIndex] of snapshot.edges) {
+    const from = nameByStopIndex[fromIndex]
+    const to = nameByStopIndex[toIndex]
+    if (!from || !to || from === to) continue
+    records.get(from).neighbours.add(to)
+    records.get(to).neighbours.add(from)
+  }
+
+  const scored = [...records.values()].map((record) => ({
+    ...record,
+    score:
+      record.modes.size * 40 +
+      record.routes.size * 12 +
+      Math.log2(record.trainIds.size + 1) * 3 +
+      Math.min(record.neighbours.size, 10),
+  }))
+  return scored
+    .sort((first, second) =>
+      second.score - first.score ||
+      second.modes.size - first.modes.size ||
+      second.routes.size - first.routes.size ||
+      second.trainIds.size - first.trainIds.size ||
+      second.neighbours.size - first.neighbours.size ||
+      first.name.localeCompare(second.name, 'en-GB'))
+    .map((record, labelRank) => ({
+      name: record.name,
+      labelRank,
+      modeCount: record.modes.size,
+      routeCount: record.routes.size,
+      movementCount: record.trainIds.size,
+      neighbourCount: record.neighbours.size,
+      score: Number(record.score.toFixed(3)),
+    }))
+}
+
+export function applyStationLabelRanks(snapshot, options) {
+  const ranking = rankNetworkStations(snapshot, options)
+  const rankByName = new Map(ranking.map(({ name, labelRank }) => [name, labelRank]))
+  return {
+    ...snapshot,
+    metadata: {
+      ...snapshot.metadata,
+      labelHierarchy: {
+        model: `Stable station-name rank by mode interchange, distinct routes, scheduled calls and graph degree${options?.catalogue ? ', enriched by advertised topology' : ''}`,
+        stationCount: ranking.length,
+      },
+    },
+    stops: snapshot.stops.map((stop) => [
+      stop[0], stop[1], stop[2], stop[3], stop[4], rankByName.get(stop[2]),
+    ]),
+  }
+}