gtfs-sqljs 0.2.2 → 0.3.1

package/README.md

@@ -1,5 +1,5 @@
 <div align="center">
-  <img src="logo.svg" alt="gtfs-sqljs logo" width="200" height="200">
+  <img src="illustrations/logo.svg" alt="gtfs-sqljs logo" width="200" height="200">
   <h1>gtfs-sqljs</h1>
 
   [![npm version](https://img.shields.io/npm/v/gtfs-sqljs)](https://www.npmjs.com/package/gtfs-sqljs)
@@ -97,7 +97,8 @@ Full API documentation: [API Reference](https://sysdevrun.github.io/gtfs-sqljs/d
 
 ### Static Methods
 
-- `GtfsSqlJs.fromZip(zipPath, options?)` - Create instance from GTFS ZIP file
+- `GtfsSqlJs.fromZip(zipPath, options?)` - Create instance from GTFS ZIP file path or URL
+- `GtfsSqlJs.fromZipData(zipData, options?)` - Create instance from pre-loaded GTFS ZIP data (`ArrayBuffer` or `Uint8Array`)
 - `GtfsSqlJs.fromDatabase(database, options?)` - Create instance from existing database
 
 ### Instance Methods

package/dist/index.d.ts

@@ -292,8 +292,8 @@ interface Agency {
 interface Stop {
     stop_id: string;
     stop_name: string;
-    stop_lat: number;
-    stop_lon: number;
+    stop_lat?: number;
+    stop_lon?: number;
     stop_code?: string;
     stop_desc?: string;
     zone_id?: string;
@@ -310,8 +310,8 @@ interface Stop {
  */
 interface Route {
     route_id: string;
-    route_short_name: string;
-    route_long_name: string;
+    route_short_name?: string;
+    route_long_name?: string;
     route_type: number;
     agency_id?: string;
     route_desc?: string;
@@ -342,8 +342,8 @@ interface Trip {
  */
 interface StopTime {
     trip_id: string;
-    arrival_time: string;
-    departure_time: string;
+    arrival_time?: string;
+    departure_time?: string;
     stop_id: string;
     stop_sequence: number;
     stop_headsign?: string;
@@ -426,7 +426,7 @@ interface Frequency {
 interface Transfer {
     from_stop_id: string;
     to_stop_id: string;
-    transfer_type: number;
+    transfer_type?: number;
     min_transfer_time?: number;
 }
 /**
@@ -714,19 +714,25 @@ declare class GtfsSqlJs {
      */
     private constructor();
     /**
-     * Create GtfsSqlJs instance from GTFS ZIP file
+     * Create GtfsSqlJs instance from GTFS ZIP file path or URL
      */
     static fromZip(zipPath: string, options?: Omit<GtfsSqlJsOptions, 'zipPath' | 'database'>): Promise<GtfsSqlJs>;
+    /**
+     * Create GtfsSqlJs instance from pre-loaded GTFS ZIP data
+     * @param source - Optional original path/URL, used for cache key generation and metadata
+     */
+    static fromZipData(zipData: ArrayBuffer | Uint8Array, options?: Omit<GtfsSqlJsOptions, 'zipPath' | 'database'>, source?: string): Promise<GtfsSqlJs>;
     /**
      * Create GtfsSqlJs instance from existing SQLite database
      */
     static fromDatabase(database: ArrayBuffer, options?: Omit<GtfsSqlJsOptions, 'zipPath' | 'database'>): Promise<GtfsSqlJs>;
     /**
-     * Initialize from ZIP file
+     * Initialize from pre-loaded ZIP data
+     * @param source - Optional original path/URL, used for cache key generation and metadata
      */
-    private initFromZip;
+    private initFromZipData;
     /**
-     * Helper method to load GTFS data from zip data (ArrayBuffer)
+     * Helper method to load GTFS data from zip data
      * Used by both cache-enabled and cache-disabled paths
      */
     private loadFromZipData;
@@ -1058,8 +1064,8 @@ interface IndexDefinition {
 declare const GTFS_SCHEMA: TableSchema[];
 
 /**
- * Compute SHA-256 checksum of data
- * Uses Web Crypto API (available in both browser and Node.js 18+)
+ * Compute SHA-256 checksum of data.
+ * Uses the global Web Crypto API (available in browsers and Node.js 18+).
  */
 declare function computeChecksum(data: ArrayBuffer | Uint8Array): Promise<string>;
 /**

package/dist/index.js

@@ -69,8 +69,8 @@ var GTFS_SCHEMA = [
     columns: [
       { name: "stop_id", type: "TEXT", required: true, primaryKey: true },
       { name: "stop_name", type: "TEXT", required: true },
-      { name: "stop_lat", type: "REAL", required: true },
-      { name: "stop_lon", type: "REAL", required: true },
+      { name: "stop_lat", type: "REAL", required: false },
+      { name: "stop_lon", type: "REAL", required: false },
       { name: "stop_code", type: "TEXT", required: false },
       { name: "stop_desc", type: "TEXT", required: false },
       { name: "zone_id", type: "TEXT", required: false },
@@ -92,8 +92,8 @@ var GTFS_SCHEMA = [
     name: "routes",
     columns: [
       { name: "route_id", type: "TEXT", required: true, primaryKey: true },
-      { name: "route_short_name", type: "TEXT", required: true },
-      { name: "route_long_name", type: "TEXT", required: true },
+      { name: "route_short_name", type: "TEXT", required: false },
+      { name: "route_long_name", type: "TEXT", required: false },
       { name: "route_type", type: "INTEGER", required: true },
       { name: "agency_id", type: "TEXT", required: false },
       { name: "route_desc", type: "TEXT", required: false },
@@ -132,8 +132,8 @@ var GTFS_SCHEMA = [
     name: "stop_times",
     columns: [
       { name: "trip_id", type: "TEXT", required: true },
-      { name: "arrival_time", type: "TEXT", required: true },
-      { name: "departure_time", type: "TEXT", required: true },
+      { name: "arrival_time", type: "TEXT", required: false },
+      { name: "departure_time", type: "TEXT", required: false },
       { name: "stop_id", type: "TEXT", required: true },
       { name: "stop_sequence", type: "INTEGER", required: true },
       { name: "stop_headsign", type: "TEXT", required: false },
@@ -236,7 +236,7 @@ var GTFS_SCHEMA = [
     columns: [
       { name: "from_stop_id", type: "TEXT", required: true },
       { name: "to_stop_id", type: "TEXT", required: true },
-      { name: "transfer_type", type: "INTEGER", required: true },
+      { name: "transfer_type", type: "INTEGER", required: false },
       { name: "min_transfer_time", type: "INTEGER", required: false }
     ],
     indexes: [
@@ -340,25 +340,28 @@ function getAllCreateIndexStatements() {
 
 // src/loaders/zip-loader.ts
 import JSZip from "jszip";
-async function loadGTFSZip(source) {
-  let zipData;
-  if (typeof source === "string") {
-    zipData = await fetchZip(source);
-  } else {
-    zipData = source;
-  }
+
+// src/utils/env.ts
+function isNodeEnvironment() {
+  return typeof process !== "undefined" && process.versions != null && process.versions.node != null;
+}
+
+// src/loaders/zip-loader.ts
+async function loadGTFSZip(zipData, skipFiles) {
   const zip = await JSZip.loadAsync(zipData);
+  const skipSet = skipFiles ? new Set(skipFiles) : null;
   const files = {};
   const filePromises = [];
   zip.forEach((relativePath, file) => {
-    if (!file.dir && relativePath.endsWith(".txt")) {
-      const fileName = relativePath.split("/").pop() || relativePath;
-      filePromises.push(
-        file.async("string").then((content) => {
-          files[fileName] = content;
-        })
-      );
-    }
+    if (file.dir) return;
+    const fileName = relativePath.split("/").pop() || relativePath;
+    if (!(fileName in GTFS_FILE_MAPPING)) return;
+    if (skipSet?.has(fileName)) return;
+    filePromises.push(
+      file.async("string").then((content) => {
+        files[fileName] = content;
+      })
+    );
   });
   await Promise.all(filePromises);
   return files;
@@ -411,8 +414,7 @@ async function fetchZip(source, onProgress) {
     }
     throw new Error("fetch is not available to load URL");
   }
-  const isNode = typeof process !== "undefined" && process.versions != null && process.versions.node != null;
-  if (isNode) {
+  if (isNodeEnvironment()) {
     try {
       const fs = await import("fs");
       const buffer = await fs.promises.readFile(source);
@@ -430,6 +432,24 @@ async function fetchZip(source, onProgress) {
   }
   throw new Error("No method available to load ZIP file");
 }
+var GTFS_FILE_MAPPING = {
+  "agency.txt": "agency",
+  "stops.txt": "stops",
+  "routes.txt": "routes",
+  "trips.txt": "trips",
+  "stop_times.txt": "stop_times",
+  "calendar.txt": "calendar",
+  "calendar_dates.txt": "calendar_dates",
+  "fare_attributes.txt": "fare_attributes",
+  "fare_rules.txt": "fare_rules",
+  "shapes.txt": "shapes",
+  "frequencies.txt": "frequencies",
+  "transfers.txt": "transfers",
+  "pathways.txt": "pathways",
+  "levels.txt": "levels",
+  "feed_info.txt": "feed_info",
+  "attributions.txt": "attributions"
+};
 
 // src/loaders/csv-parser.ts
 import Papa from "papaparse";
@@ -867,8 +887,7 @@ async function fetchProtobuf(source) {
     const arrayBuffer2 = await response2.arrayBuffer();
     return new Uint8Array(arrayBuffer2);
   }
-  const isNode = typeof process !== "undefined" && process.versions != null && process.versions.node != null;
-  if (isNode) {
+  if (isNodeEnvironment()) {
     try {
       const fs = await import("fs");
       const buffer = await fs.promises.readFile(source);
@@ -898,24 +917,13 @@ function loadGtfsRtProto() {
 function camelToSnake(str) {
   return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
 }
-function convertKeysToSnakeCase(obj) {
-  if (obj === null || obj === void 0) {
-    return obj;
+function convertObjectKeysToSnakeCase(obj) {
+  const result = {};
+  for (const [key, value] of Object.entries(obj)) {
+    const snakeKey = camelToSnake(key);
+    result[snakeKey] = value !== null && typeof value === "object" && !Array.isArray(value) ? convertObjectKeysToSnakeCase(value) : value;
   }
-  if (Array.isArray(obj)) {
-    return obj.map(convertKeysToSnakeCase);
-  }
-  if (typeof obj === "object") {
-    const result = {};
-    for (const key in obj) {
-      if (Object.prototype.hasOwnProperty.call(obj, key)) {
-        const snakeKey = camelToSnake(key);
-        result[snakeKey] = convertKeysToSnakeCase(obj[key]);
-      }
-    }
-    return result;
-  }
-  return obj;
+  return result;
 }
 function parseTranslatedString(ts) {
   if (!ts || !ts.translation || ts.translation.length === 0) {
@@ -936,8 +944,8 @@ function insertAlerts(db, alerts, timestamp) {
     ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
   for (const alert of alerts) {
-    const activePeriodSnake = convertKeysToSnakeCase(alert.activePeriod || []);
-    const informedEntitySnake = convertKeysToSnakeCase(alert.informedEntity || []);
+    const activePeriodSnake = (alert.activePeriod ?? []).map(convertObjectKeysToSnakeCase);
+    const informedEntitySnake = (alert.informedEntity ?? []).map(convertObjectKeysToSnakeCase);
     stmt.run([
       alert.id,
       JSON.stringify(activePeriodSnake),
@@ -1098,24 +1106,16 @@ async function loadRealtimeData(db, feedUrls) {
 }
 
 // src/cache/checksum.ts
-async function getCrypto() {
-  if (typeof crypto !== "undefined" && crypto.subtle) {
-    return crypto;
-  }
-  if (typeof globalThis !== "undefined") {
-    try {
-      const { webcrypto } = await import("crypto");
-      return webcrypto;
-    } catch {
-      throw new Error("Web Crypto API not available");
-    }
-  }
-  throw new Error("Crypto not available in this environment");
-}
 async function computeChecksum(data) {
-  const cryptoInstance = await getCrypto();
-  const bufferSource = data instanceof Uint8Array ? data : new Uint8Array(data);
-  const hashBuffer = await cryptoInstance.subtle.digest("SHA-256", bufferSource);
+  let buffer;
+  if (data instanceof ArrayBuffer) {
+    buffer = data;
+  } else {
+    const copy = new ArrayBuffer(data.byteLength);
+    new Uint8Array(copy).set(data);
+    buffer = copy;
+  }
+  const hashBuffer = await crypto.subtle.digest("SHA-256", buffer);
   const hashArray = Array.from(new Uint8Array(hashBuffer));
   const hashHex = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
   return hashHex;
@@ -1257,8 +1257,8 @@ function rowToStop(row) {
   return {
     stop_id: String(row.stop_id),
     stop_name: String(row.stop_name),
-    stop_lat: Number(row.stop_lat),
-    stop_lon: Number(row.stop_lon),
+    stop_lat: row.stop_lat !== null ? Number(row.stop_lat) : void 0,
+    stop_lon: row.stop_lon !== null ? Number(row.stop_lon) : void 0,
     stop_code: row.stop_code ? String(row.stop_code) : void 0,
     stop_desc: row.stop_desc ? String(row.stop_desc) : void 0,
     zone_id: row.zone_id ? String(row.zone_id) : void 0,
@@ -1317,8 +1317,8 @@ function getRoutes(db, filters = {}) {
 function rowToRoute(row) {
   return {
     route_id: String(row.route_id),
-    route_short_name: String(row.route_short_name),
-    route_long_name: String(row.route_long_name),
+    route_short_name: row.route_short_name ? String(row.route_short_name) : void 0,
+    route_long_name: row.route_long_name ? String(row.route_long_name) : void 0,
     route_type: Number(row.route_type),
     agency_id: row.agency_id ? String(row.agency_id) : void 0,
     route_desc: row.route_desc ? String(row.route_desc) : void 0,
@@ -1810,7 +1810,7 @@ function buildOrderedStopList(db, tripIds) {
     if (!tripStopSequences.has(tripId)) {
       tripStopSequences.set(tripId, []);
     }
-    tripStopSequences.get(tripId).push({ stop_id: stopId, stop_sequence: stopSequence });
+    tripStopSequences.get(tripId)?.push({ stop_id: stopId, stop_sequence: stopSequence });
   }
   stmt.free();
   const orderedStopIds = [];
@@ -1868,8 +1868,8 @@ function findInsertionPosition(orderedStopIds, newStopId, tripStops, currentInde
 function rowToStopTime(row) {
   return {
     trip_id: String(row.trip_id),
-    arrival_time: String(row.arrival_time),
-    departure_time: String(row.departure_time),
+    arrival_time: row.arrival_time ? String(row.arrival_time) : void 0,
+    departure_time: row.departure_time ? String(row.departure_time) : void 0,
     stop_id: String(row.stop_id),
     stop_sequence: Number(row.stop_sequence),
     stop_headsign: row.stop_headsign ? String(row.stop_headsign) : void 0,
@@ -2308,7 +2308,7 @@ function getTripUpdates(db, filters = {}, stalenessThreshold = 120) {
       if (!stopTimesByTripId.has(stu.trip_id)) {
         stopTimesByTripId.set(stu.trip_id, []);
       }
-      stopTimesByTripId.get(stu.trip_id).push(stu);
+      stopTimesByTripId.get(stu.trip_id)?.push(stu);
     }
     for (const tu of tripUpdates) {
       tu.stop_time_update = stopTimesByTripId.get(tu.trip_id) || [];
@@ -2334,7 +2334,7 @@ function getAllTripUpdates(db) {
       if (!stopTimesByTripId.has(stu.trip_id)) {
         stopTimesByTripId.set(stu.trip_id, []);
       }
-      stopTimesByTripId.get(stu.trip_id).push(stu);
+      stopTimesByTripId.get(stu.trip_id)?.push(stu);
     }
     for (const tu of tripUpdates) {
       tu.stop_time_update = stopTimesByTripId.get(tu.trip_id) || [];
@@ -2357,11 +2357,19 @@ var GtfsSqlJs = class _GtfsSqlJs {
     this.lastRealtimeFetchTimestamp = null;
   }
   /**
-   * Create GtfsSqlJs instance from GTFS ZIP file
+   * Create GtfsSqlJs instance from GTFS ZIP file path or URL
    */
   static async fromZip(zipPath, options = {}) {
+    const zipData = await fetchZip(zipPath, options.onProgress);
+    return _GtfsSqlJs.fromZipData(zipData, options, zipPath);
+  }
+  /**
+   * Create GtfsSqlJs instance from pre-loaded GTFS ZIP data
+   * @param source - Optional original path/URL, used for cache key generation and metadata
+   */
+  static async fromZipData(zipData, options = {}, source) {
     const instance = new _GtfsSqlJs();
-    await instance.initFromZip(zipPath, options);
+    await instance.initFromZipData(zipData, options, source);
     return instance;
   }
   /**
@@ -2373,9 +2381,10 @@ var GtfsSqlJs = class _GtfsSqlJs {
     return instance;
   }
   /**
-   * Initialize from ZIP file
+   * Initialize from pre-loaded ZIP data
+   * @param source - Optional original path/URL, used for cache key generation and metadata
    */
-  async initFromZip(zipPath, options) {
+  async initFromZipData(zipData, options, source) {
     const onProgress = options.onProgress;
     const {
       cache: userCache,
@@ -2396,20 +2405,14 @@ var GtfsSqlJs = class _GtfsSqlJs {
         percentComplete: 0,
         message: "Checking cache..."
       });
-      let zipData2;
-      if (typeof zipPath === "string") {
-        zipData2 = await fetchZip(zipPath, onProgress);
-      } else {
-        zipData2 = zipPath;
-      }
-      const filesize = zipData2.byteLength;
-      const checksum = await computeZipChecksum(zipData2);
+      const filesize = zipData.byteLength;
+      const checksum = await computeZipChecksum(zipData);
       const cacheKey = generateCacheKey(
         checksum,
         LIB_VERSION,
         cacheVersion,
         filesize,
-        typeof zipPath === "string" ? zipPath : void 0,
+        source,
         skipFiles
       );
       const cacheEntry = await cache.get(cacheKey);
@@ -2458,7 +2461,7 @@ var GtfsSqlJs = class _GtfsSqlJs {
           return;
         }
       }
-      await this.loadFromZipData(zipData2, options, onProgress);
+      await this.loadFromZipData(zipData, options, onProgress);
       onProgress?.({
         phase: "saving_cache",
         currentFile: null,
@@ -2474,7 +2477,7 @@ var GtfsSqlJs = class _GtfsSqlJs {
         checksum,
         version: cacheVersion,
         timestamp: Date.now(),
-        source: typeof zipPath === "string" ? zipPath : void 0,
+        source,
         size: dbBuffer.byteLength,
         skipFiles
       });
@@ -2490,12 +2493,6 @@ var GtfsSqlJs = class _GtfsSqlJs {
       });
       return;
     }
-    let zipData;
-    if (typeof zipPath === "string") {
-      zipData = await fetchZip(zipPath, onProgress);
-    } else {
-      zipData = zipPath;
-    }
     await this.loadFromZipData(zipData, options, onProgress);
     onProgress?.({
       phase: "complete",
@@ -2509,10 +2506,13 @@ var GtfsSqlJs = class _GtfsSqlJs {
     });
   }
   /**
-   * Helper method to load GTFS data from zip data (ArrayBuffer)
+   * Helper method to load GTFS data from zip data
    * Used by both cache-enabled and cache-disabled paths
    */
   async loadFromZipData(zipData, options, onProgress) {
+    if (!this.SQL) {
+      throw new Error("SQL.js not initialized");
+    }
     this.db = new this.SQL.Database();
     this.db.run("PRAGMA synchronous = OFF");
     this.db.run("PRAGMA journal_mode = MEMORY");
@@ -2544,7 +2544,7 @@ var GtfsSqlJs = class _GtfsSqlJs {
       percentComplete: 35,
       message: "Extracting GTFS ZIP file"
     });
-    const files = await loadGTFSZip(zipData);
+    const files = await loadGTFSZip(zipData, options.skipFiles);
     onProgress?.({
       phase: "inserting_data",
       currentFile: null,