@osmix/gtfs 0.0.1

package/src/gtfs-archive.ts

@@ -0,0 +1,138 @@
+/**
+ * Lazy GTFS archive parser with streaming CSV support.
+ *
+ * Only parses CSV files when they are accessed, not upfront.
+ * Uses @std/csv CsvParseStream for true line-by-line streaming.
+ *
+ * @module
+ */
+
+import { CsvParseStream } from "@std/csv/parse-stream"
+import { unzip, type ZipItem } from "but-unzip"
+import type {
+	GtfsAgency,
+	GtfsRoute,
+	GtfsShapePoint,
+	GtfsStop,
+	GtfsStopTime,
+	GtfsTrip,
+} from "./types"
+import { bytesToTextStream } from "./utils"
+
+/**
+ * Map of GTFS filenames to their record types.
+ */
+export interface GtfsFileTypeMap {
+	"agency.txt": GtfsAgency
+	"stops.txt": GtfsStop
+	"routes.txt": GtfsRoute
+	"trips.txt": GtfsTrip
+	"stop_times.txt": GtfsStopTime
+	"shapes.txt": GtfsShapePoint
+}
+
+/** Valid GTFS filenames that can be parsed. */
+export type GtfsFileName = keyof GtfsFileTypeMap
+
+/**
+ * Lazy GTFS archive that only parses files on demand.
+ *
+ * Files are read from the zip and parsed only when their
+ * corresponding getter is called for the first time.
+ * Streaming iterators parse CSV line-by-line without loading
+ * the entire file into memory.
+ */
+export class GtfsArchive {
+	private entries: Map<string, ZipItem>
+
+	private constructor(entries: Map<string, ZipItem>) {
+		this.entries = entries
+	}
+
+	/**
+	 * Create a GtfsArchive from zip data.
+	 */
+	static fromZip(zipData: ArrayBuffer | Uint8Array): GtfsArchive {
+		const bytes =
+			zipData instanceof Uint8Array ? zipData : new Uint8Array(zipData)
+		const items = unzip(bytes)
+
+		const entries = new Map<string, ZipItem>()
+		for (const item of items) {
+			// Remove directory prefix and store by filename
+			const name = item.filename.replace(/^.*\//, "")
+			if (name.endsWith(".txt")) {
+				entries.set(name, item)
+			}
+		}
+
+		return new GtfsArchive(entries)
+	}
+
+	/**
+	 * Check if a file exists in the archive.
+	 */
+	hasFile(filename: string): boolean {
+		return this.entries.has(filename)
+	}
+
+	/**
+	 * List all files in the archive.
+	 */
+	listFiles(): string[] {
+		return Array.from(this.entries.keys())
+	}
+
+	/**
+	 * Get a readable stream of bytes for a file.
+	 */
+	private async getFileBytes(filename: string): Promise<Uint8Array | null> {
+		const entry = this.entries.get(filename)
+		if (!entry) return null
+
+		const data = entry.read()
+		return data instanceof Promise ? await data : data
+	}
+
+	/**
+	 * Stream parse a CSV file, yielding typed records one at a time.
+	 *
+	 * The return type is automatically inferred based on the filename:
+	 * - `"stops.txt"` → `AsyncGenerator<GtfsStop>`
+	 * - `"routes.txt"` → `AsyncGenerator<GtfsRoute>`
+	 * - `"shapes.txt"` → `AsyncGenerator<GtfsShapePoint>`
+	 * - etc.
+	 *
+	 * @param filename - The GTFS filename to parse (e.g., "stops.txt")
+	 * @returns An async generator yielding typed records
+	 *
+	 * @example
+	 * ```ts
+	 * for await (const stop of archive.iter("stops.txt")) {
+	 *   console.log(stop.stop_name) // TypeScript knows this is GtfsStop
+	 * }
+	 * ```
+	 */
+	async *iter<F extends GtfsFileName>(
+		filename: F,
+	): AsyncGenerator<GtfsFileTypeMap[F], void, unknown> {
+		const bytes = await this.getFileBytes(filename)
+		if (!bytes) return
+
+		const textStream = bytesToTextStream(bytes)
+		const csvStream = textStream.pipeThrough(
+			new CsvParseStream({ skipFirstRow: true }),
+		)
+
+		const reader = csvStream.getReader()
+		try {
+			while (true) {
+				const { value, done } = await reader.read()
+				if (done) break
+				yield value as unknown as GtfsFileTypeMap[F]
+			}
+		} finally {
+			reader.releaseLock()
+		}
+	}
+}

package/src/index.ts

@@ -0,0 +1,54 @@
+/**
+ * @osmix/gtfs - Convert GTFS transit feeds to OSM format.
+ *
+ * Parses zipped GTFS files lazily and converts transit data to OpenStreetMap entities:
+ * - **Stops** become **Nodes** with public transport tags
+ * - **Routes** become **Ways** with shape geometry
+ *
+ * Files are only parsed when needed, not upfront.
+ *
+ * @example
+ * ```ts
+ * import { fromGtfs } from "@osmix/gtfs"
+ *
+ * const response = await fetch("https://example.com/gtfs.zip")
+ * const zipData = await response.arrayBuffer()
+ * const osm = await fromGtfs(zipData, { id: "transit" })
+ *
+ * console.log(`Imported ${osm.nodes.size} stops and ${osm.ways.size} routes`)
+ * ```
+ *
+ * @example Using GtfsArchive directly for custom processing
+ * ```ts
+ * import { GtfsArchive } from "@osmix/gtfs"
+ *
+ * const archive = GtfsArchive.fromZip(zipData)
+ *
+ * // Only parse stops - other files remain unread
+ * for await (const stop of archive.iterStops()) {
+ *   console.log(stop.stop_name)
+ * }
+ * ```
+ *
+ * @module @osmix/gtfs
+ */
+
+export { fromGtfs, GtfsOsmBuilder } from "./from-gtfs"
+export {
+	GtfsArchive,
+	type GtfsFileName,
+	type GtfsFileTypeMap,
+} from "./gtfs-archive"
+export {
+	type GtfsAgency,
+	type GtfsConversionOptions,
+	type GtfsFeed,
+	type GtfsRoute,
+	type GtfsShapePoint,
+	type GtfsStop,
+	type GtfsStopTime,
+	type GtfsTrip,
+	routeTypeToOsmRoute,
+	wheelchairBoardingToOsm,
+} from "./types"
+export { isGtfsZip } from "./utils"

package/src/types.ts

@@ -0,0 +1,184 @@
+/**
+ * GTFS (General Transit Feed Specification) type definitions.
+ *
+ * @module
+ */
+
+/**
+ * GTFS stop from stops.txt.
+ * Represents a transit stop or station.
+ */
+export interface GtfsStop {
+	stop_id: string
+	stop_code?: string
+	stop_name: string
+	stop_desc?: string
+	stop_lat: string
+	stop_lon: string
+	zone_id?: string
+	stop_url?: string
+	/** 0 = stop, 1 = station, 2 = entrance/exit, 3 = generic node, 4 = boarding area */
+	location_type?: string
+	parent_station?: string
+	stop_timezone?: string
+	/** 0 = no info, 1 = accessible, 2 = not accessible */
+	wheelchair_boarding?: string
+	level_id?: string
+	platform_code?: string
+}
+
+/**
+ * GTFS route from routes.txt.
+ * Represents a transit route/line.
+ */
+export interface GtfsRoute {
+	route_id: string
+	agency_id?: string
+	route_short_name?: string
+	route_long_name?: string
+	route_desc?: string
+	/**
+	 * Route type:
+	 * 0 = Tram, 1 = Subway, 2 = Rail, 3 = Bus, 4 = Ferry,
+	 * 5 = Cable tram, 6 = Aerial lift, 7 = Funicular,
+	 * 11 = Trolleybus, 12 = Monorail
+	 */
+	route_type: string
+	route_url?: string
+	route_color?: string
+	route_text_color?: string
+	route_sort_order?: string
+	continuous_pickup?: string
+	continuous_drop_off?: string
+	network_id?: string
+}
+
+/**
+ * GTFS shape point from shapes.txt.
+ * Defines the geographic path of a route.
+ */
+export interface GtfsShapePoint {
+	shape_id: string
+	shape_pt_lat: string
+	shape_pt_lon: string
+	shape_pt_sequence: string
+	shape_dist_traveled?: string
+}
+
+/**
+ * GTFS trip from trips.txt.
+ * Represents a specific trip on a route.
+ */
+export interface GtfsTrip {
+	trip_id: string
+	route_id: string
+	service_id: string
+	trip_headsign?: string
+	trip_short_name?: string
+	direction_id?: string
+	block_id?: string
+	shape_id?: string
+	wheelchair_accessible?: string
+	bikes_allowed?: string
+}
+
+/**
+ * GTFS stop time from stop_times.txt.
+ * Links trips to stops with timing info.
+ */
+export interface GtfsStopTime {
+	trip_id: string
+	arrival_time?: string
+	departure_time?: string
+	stop_id: string
+	stop_sequence: string
+	stop_headsign?: string
+	pickup_type?: string
+	drop_off_type?: string
+	continuous_pickup?: string
+	continuous_drop_off?: string
+	shape_dist_traveled?: string
+	timepoint?: string
+}
+
+/**
+ * GTFS agency from agency.txt.
+ */
+export interface GtfsAgency {
+	agency_id?: string
+	agency_name: string
+	agency_url: string
+	agency_timezone: string
+	agency_lang?: string
+	agency_phone?: string
+	agency_fare_url?: string
+	agency_email?: string
+}
+
+/**
+ * Parsed GTFS feed with all relevant files.
+ */
+export interface GtfsFeed {
+	agencies: GtfsAgency[]
+	stops: GtfsStop[]
+	routes: GtfsRoute[]
+	trips: GtfsTrip[]
+	stopTimes: GtfsStopTime[]
+	shapes: GtfsShapePoint[]
+}
+
+/**
+ * Options for GTFS to OSM conversion.
+ */
+export interface GtfsConversionOptions {
+	/** Whether to include stops as nodes. Default: true */
+	includeStops?: boolean
+	/** Whether to include routes as ways. Default: true */
+	includeRoutes?: boolean
+}
+
+/**
+ * Map GTFS route_type to OSM route tag value.
+ */
+export function routeTypeToOsmRoute(routeType: string): string {
+	switch (routeType) {
+		case "0":
+			return "tram"
+		case "1":
+			return "subway"
+		case "2":
+			return "train"
+		case "3":
+			return "bus"
+		case "4":
+			return "ferry"
+		case "5":
+			return "tram" // Cable tram
+		case "6":
+			return "aerialway"
+		case "7":
+			return "funicular"
+		case "11":
+			return "trolleybus"
+		case "12":
+			return "train" // Monorail
+		default:
+			return "bus"
+	}
+}
+
+/**
+ * Map GTFS wheelchair_boarding to OSM wheelchair tag value.
+ */
+export function wheelchairBoardingToOsm(
+	value: string | undefined,
+): string | undefined {
+	switch (value) {
+		case "1":
+			return "yes"
+		case "2":
+			return "no"
+		default:
+			return undefined
+	}
+}

package/src/utils.ts

@@ -0,0 +1,226 @@
+import { normalizeHexColor } from "@osmix/shared/color"
+import type { OsmTags } from "@osmix/shared/types"
+import {
+	type GtfsRoute,
+	type GtfsStop,
+	type GtfsTrip,
+	routeTypeToOsmRoute,
+	wheelchairBoardingToOsm,
+} from "./types"
+
+/**
+ * Check if a ZIP file (as bytes) is a GTFS archive by looking for characteristic GTFS files.
+ * GTFS archives must contain at least agency.txt, stops.txt, routes.txt, trips.txt,
+ * and stop_times.txt according to the GTFS specification.
+ */
+export function isGtfsZip(bytes: Uint8Array): boolean {
+	// GTFS required files per the spec
+	const requiredGtfsFiles = [
+		"agency.txt",
+		"stops.txt",
+		"routes.txt",
+		"trips.txt",
+		"stop_times.txt",
+	]
+
+	// Find filenames in the ZIP by scanning for the local file headers
+	// ZIP local file header signature: 0x04034b50 (little-endian: 50 4b 03 04)
+	const foundFiles = new Set<string>()
+	const decoder = new TextDecoder()
+	let pos = 0
+
+	while (pos < bytes.length - 30) {
+		// Check for ZIP local file header signature
+		const isLocalHeader =
+			bytes[pos] === 0x50 &&
+			bytes[pos + 1] === 0x4b &&
+			bytes[pos + 2] === 0x03 &&
+			bytes[pos + 3] === 0x04
+
+		if (isLocalHeader) {
+			// Read filename length at offset 26-27 (little-endian)
+			const nameLen = (bytes[pos + 26] ?? 0) | ((bytes[pos + 27] ?? 0) << 8)
+			// Read extra field length at offset 28-29 (little-endian)
+			const extraLen = (bytes[pos + 28] ?? 0) | ((bytes[pos + 29] ?? 0) << 8)
+
+			// Defensive bounds check
+			if (nameLen < 0 || extraLen < 0) break
+			if (pos + 30 + nameLen + extraLen > bytes.length) break
+
+			// General purpose bit flag at offset 6-7 (little-endian)
+			// Bit 3 indicates the presence of a data descriptor, in which case
+			// the compressed size fields in the local header are zero and the
+			// actual sizes follow the compressed data.
+			const flags = (bytes[pos + 6] ?? 0) | ((bytes[pos + 7] ?? 0) << 8)
+			const hasDataDescriptor = (flags & 0x0008) !== 0
+
+			// Extract filename (starts at offset 30)
+			const nameBytes = bytes.slice(pos + 30, pos + 30 + nameLen)
+			const filename = decoder.decode(nameBytes)
+
+			// Normalize path - extract just the filename part
+			const basename = filename.replace(/^.*\//, "").toLowerCase()
+			if (basename) {
+				foundFiles.add(basename)
+			}
+
+			if (!hasDataDescriptor) {
+				// Read compressed size at offset 18-21 (little-endian).
+				// Use >>> 0 to ensure unsigned 32-bit interpretation, since
+				// JavaScript bitwise operations return signed 32-bit integers.
+				// Without this, files >= 2GB would produce negative values.
+				const compSize =
+					((bytes[pos + 18] ?? 0) |
+						((bytes[pos + 19] ?? 0) << 8) |
+						((bytes[pos + 20] ?? 0) << 16) |
+						((bytes[pos + 21] ?? 0) << 24)) >>>
+					0
+
+				// Move to next entry using the known compressed size
+				pos += 30 + nameLen + extraLen + compSize
+			} else {
+				// When a data descriptor is present, the compressed size is not
+				// available in the local header. Skip past the header, filename,
+				// and extra fields, then scan forward for the next local header
+				// signature. This avoids getting stuck at the same position when
+				// the compressed size field is zero.
+				pos += 30 + nameLen + extraLen
+
+				while (pos < bytes.length - 3) {
+					const nextIsHeader =
+						bytes[pos] === 0x50 &&
+						bytes[pos + 1] === 0x4b &&
+						bytes[pos + 2] === 0x03 &&
+						bytes[pos + 3] === 0x04
+					if (nextIsHeader) break
+					pos++
+				}
+			}
+		} else {
+			pos++
+		}
+	}
+
+	// Check if all required GTFS files are present
+	return requiredGtfsFiles.every((f) => foundFiles.has(f))
+}
+
+/**
+ * Convert a GTFS stop to OSM tags.
+ */
+export function stopToTags(stop: GtfsStop): OsmTags {
+	const tags: OsmTags = {
+		public_transport: "platform",
+	}
+
+	if (stop.stop_name) tags["name"] = stop.stop_name
+	if (stop.stop_id) tags["ref"] = stop.stop_id
+	if (stop.stop_code) tags["ref:gtfs:stop_code"] = stop.stop_code
+	if (stop.stop_desc) tags["description"] = stop.stop_desc
+	if (stop.stop_url) tags["website"] = stop.stop_url
+	if (stop.platform_code) tags["ref:platform"] = stop.platform_code
+
+	// Location type determines more specific tagging
+	const locationType = stop.location_type ?? "0"
+	switch (locationType) {
+		case "1":
+			tags["public_transport"] = "station"
+			break
+		case "2":
+			// Entrances are not platforms - remove the default and use railway tag
+			delete tags["public_transport"]
+			tags["railway"] = "subway_entrance"
+			break
+		case "3":
+			// Generic node - keep as platform
+			break
+		case "4":
+			tags["public_transport"] = "platform"
+			break
+	}
+
+	// Wheelchair accessibility
+	const wheelchair = wheelchairBoardingToOsm(stop.wheelchair_boarding)
+	if (wheelchair) tags["wheelchair"] = wheelchair
+
+	return tags
+}
+
+/**
+ * Convert a GTFS route to OSM tags.
+ */
+export function routeToTags(route: GtfsRoute): OsmTags {
+	const tags: OsmTags = {
+		route: routeTypeToOsmRoute(route.route_type),
+	}
+
+	// Use long name if available, otherwise short name
+	if (route.route_long_name) {
+		tags["name"] = route.route_long_name
+	} else if (route.route_short_name) {
+		tags["name"] = route.route_short_name
+	}
+
+	if (route.route_short_name) tags["ref"] = route.route_short_name
+	if (route.route_id) tags["ref:gtfs:route_id"] = route.route_id
+	if (route.route_desc) tags["description"] = route.route_desc
+	if (route.route_url) tags["website"] = route.route_url
+
+	// Route color (validate and normalize hex color)
+	const color = normalizeHexColor(route.route_color)
+	if (color) tags["color"] = color
+
+	const textColor = normalizeHexColor(route.route_text_color)
+	if (textColor) tags["text_color"] = textColor
+
+	// Route type as additional tag
+	tags["gtfs:route_type"] = route.route_type
+
+	return tags
+}
+
+/**
+ * Convert a GTFS trip to OSM tags.
+ */
+export function tripToTags(trip: GtfsTrip): OsmTags {
+	const tags: OsmTags = {}
+	tags["ref:gtfs:trip_id"] = trip.trip_id
+	if (trip.service_id) tags["ref:gtfs:service_id"] = trip.service_id
+	if (trip.trip_headsign) tags["ref:gtfs:trip_headsign"] = trip.trip_headsign
+	if (trip.trip_short_name)
+		tags["ref:gtfs:trip_short_name"] = trip.trip_short_name
+	if (trip.direction_id) tags["ref:gtfs:direction_id"] = trip.direction_id
+	if (trip.block_id) tags["ref:gtfs:block_id"] = trip.block_id
+	if (trip.shape_id) tags["ref:gtfs:shape_id"] = trip.shape_id
+	if (trip.wheelchair_accessible)
+		tags["ref:gtfs:wheelchair_accessible"] = trip.wheelchair_accessible
+	if (trip.bikes_allowed) tags["ref:gtfs:bikes_allowed"] = trip.bikes_allowed
+
+	return tags
+}
+
+/**
+ * Create a ReadableStream of text from file bytes.
+ */
+export function bytesToTextStream(bytes: Uint8Array): ReadableStream<string> {
+	const decoder = new TextDecoder()
+	let offset = 0
+	const chunkSize = 64 * 1024 // 64KB chunks
+
+	return new ReadableStream<string>({
+		pull(controller) {
+			if (offset >= bytes.length) {
+				controller.close()
+				return
+			}
+
+			const end = Math.min(offset + chunkSize, bytes.length)
+			const chunk = bytes.subarray(offset, end)
+			offset = end
+
+			controller.enqueue(
+				decoder.decode(chunk, { stream: offset < bytes.length }),
+			)
+		},
+	})
+}