gtfs 4.15.6 → 4.15.8

package/dist/bin/gtfs-import.js

@@ -18,60 +18,13 @@ import { omit, snakeCase } from "lodash-es";
 import sanitize from "sanitize-filename";
 import untildify from "untildify";
 import StreamZip from "node-stream-zip";
-async function getConfig(argv2) {
-  let config;
-  let data;
-  try {
-    if (argv2.configPath) {
-      const configPath = path.resolve(untildify(argv2.configPath));
-      data = await readFile(configPath, "utf8");
-      config = Object.assign(JSON.parse(data), argv2);
-    } else if (argv2.gtfsPath || argv2.gtfsUrl || argv2.sqlitePath) {
-      const agencies = [
-        ...argv2.gtfsPath ? [{ path: argv2.gtfsPath }] : [],
-        ...argv2.gtfsUrl ? [{ url: argv2.gtfsUrl }] : []
-      ];
-      config = {
-        agencies,
-        ...omit(argv2, ["path", "url"])
-      };
-    } else if (existsSync(path.resolve("./config.json"))) {
-      data = await readFile(path.resolve("./config.json"), "utf8");
-      config = Object.assign(JSON.parse(data), argv2);
-      console.log("Using configuration from ./config.json");
-    } else {
-      throw new Error(
-        "Cannot find configuration file. Use config-sample.json as a starting point, pass --configPath option."
-      );
-    }
-    return config;
-  } catch (error) {
-    if (error instanceof SyntaxError) {
-      throw new Error(
-        `Cannot parse configuration file. Check to ensure that it is valid JSON. Error: ${error.message}`
-      );
-    }
-    throw error;
-  }
-}
-async function unzip(zipfilePath, exportPath) {
-  try {
-    const zip = new StreamZip.async({ file: zipfilePath });
-    await zip.extract(null, exportPath);
-    await zip.close();
-  } catch (error) {
-    throw new Error(
-      `Failed to extract zip file: ${error instanceof Error ? error.message : "Unknown error"}`
-    );
-  }
-}
 
 // src/lib/log-utils.ts
 import { clearLine, cursorTo } from "node:readline";
 import { noop } from "lodash-es";
 import * as colors from "yoctocolors";
 function log(config) {
-  if (!config.verbose) {
+  if (config.verbose === false) {
     return noop;
   }
   if (config.logFunction) {
@@ -116,6 +69,55 @@ function formatError(error) {
   return colors.red(`${colors.underline("Error")}: ${cleanMessage}`);
 }
 
+// src/lib/file-utils.ts
+async function getConfig(argv2) {
+  let config;
+  let data;
+  try {
+    if (argv2.configPath) {
+      const configPath = path.resolve(untildify(argv2.configPath));
+      data = await readFile(configPath, "utf8");
+      config = Object.assign(JSON.parse(data), argv2);
+    } else if (argv2.gtfsPath || argv2.gtfsUrl || argv2.sqlitePath) {
+      const agencies = [
+        ...argv2.gtfsPath ? [{ path: argv2.gtfsPath }] : [],
+        ...argv2.gtfsUrl ? [{ url: argv2.gtfsUrl }] : []
+      ];
+      config = {
+        agencies,
+        ...omit(argv2, ["path", "url"])
+      };
+    } else if (existsSync(path.resolve("./config.json"))) {
+      data = await readFile(path.resolve("./config.json"), "utf8");
+      config = Object.assign(JSON.parse(data), argv2);
+      log(config)("Using configuration from ./config.json");
+    } else {
+      throw new Error(
+        "Cannot find configuration file. Use config-sample.json as a starting point, pass --configPath option."
+      );
+    }
+    return config;
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      throw new Error(
+        `Cannot parse configuration file. Check to ensure that it is valid JSON. Error: ${error.message}`
+      );
+    }
+    throw error;
+  }
+}
+async function unzip(zipfilePath, exportPath) {
+  try {
+    const zip = new StreamZip.async({ file: zipfilePath });
+    await zip.extract(null, exportPath);
+    await zip.close();
+  } catch (error) {
+    throw new Error(
+      `Failed to extract zip file: ${error instanceof Error ? error.message : "Unknown error"}`
+    );
+  }
+}
+
 // src/lib/import-gtfs.ts
 import path2 from "node:path";
 import { createReadStream, existsSync as existsSync2, lstatSync } from "node:fs";
@@ -264,7 +266,8 @@ var attributions = {
     {
       name: "attribution_id",
       type: "text",
-      prefix: true
+      prefix: true,
+      primary: true
     },
     {
       name: "agency_id",
@@ -697,26 +700,31 @@ var fareRules = {
       name: "fare_id",
       type: "text",
       required: true,
+      primary: true,
       prefix: true
     },
     {
       name: "route_id",
       type: "text",
+      primary: true,
       prefix: true
     },
     {
       name: "origin_id",
       type: "text",
+      primary: true,
       prefix: true
     },
     {
       name: "destination_id",
       type: "text",
+      primary: true,
       prefix: true
     },
     {
       name: "contains_id",
       type: "text",
+      primary: true,
       prefix: true
     }
   ]
@@ -923,14 +931,16 @@ var locationGroupStops = {
       type: "text",
       prefix: true,
       index: true,
-      required: true
+      required: true,
+      primary: true
     },
     {
       name: "stop_id",
       type: "text",
       required: true,
       prefix: true,
-      index: true
+      index: true,
+      primary: true
     }
   ]
 };
@@ -1191,12 +1201,14 @@ var stopAreas = {
       name: "area_id",
       type: "text",
       required: true,
+      primary: true,
       prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       required: true,
+      primary: true,
       prefix: true
     }
   ]
@@ -1420,16 +1432,19 @@ var timeframes = {
     },
     {
       name: "start_time",
-      type: "text"
+      type: "text",
+      primary: true
     },
     {
       name: "end_time",
-      type: "text"
+      type: "text",
+      primary: true
     },
     {
       name: "service_id",
       type: "text",
       required: true,
+      primary: true,
       index: true,
       prefix: true
     }
@@ -1616,21 +1631,19 @@ var timetables = {
   filenameExtension: "txt",
   nonstandard: true,
   schema: [
-    {
-      name: "id",
-      type: "integer",
-      primary: true,
-      prefix: true
-    },
     {
       name: "timetable_id",
       type: "text",
-      prefix: true
+      prefix: true,
+      required: true,
+      primary: true
     },
     {
       name: "route_id",
       type: "text",
-      prefix: true
+      prefix: true,
+      required: true,
+      primary: true
     },
     {
       name: "direction_id",
@@ -1764,6 +1777,7 @@ var timetablePages = {
       name: "timetable_page_id",
       type: "text",
       primary: true,
+      required: true,
       prefix: true
     },
     {
@@ -1783,28 +1797,28 @@ var timetableStopOrder = {
   filenameExtension: "txt",
   nonstandard: true,
   schema: [
-    {
-      name: "id",
-      type: "integer",
-      primary: true,
-      prefix: true
-    },
     {
       name: "timetable_id",
       type: "text",
       index: true,
-      prefix: true
+      prefix: true,
+      required: true,
+      primary: true
     },
     {
       name: "stop_id",
       type: "text",
-      prefix: true
+      prefix: true,
+      required: true,
+      primary: true
     },
     {
       name: "stop_sequence",
       type: "integer",
       min: 0,
-      index: true
+      index: true,
+      required: true,
+      primary: true
     }
   ]
 };
@@ -1819,7 +1833,8 @@ var timetableNotes = {
       name: "note_id",
       type: "text",
       primary: true,
-      prefix: true
+      prefix: true,
+      required: true
     },
     {
       name: "symbol",
@@ -1828,7 +1843,8 @@ var timetableNotes = {
     {
       name: "note",
       type: "text",
-      nocase: true
+      nocase: true,
+      required: true
     }
   ]
 };
@@ -1842,37 +1858,39 @@ var timetableNotesReferences = {
     {
       name: "note_id",
       type: "text",
-      prefix: true
+      prefix: true,
+      required: true,
+      primary: true
     },
     {
       name: "timetable_id",
       type: "text",
-      index: true,
-      prefix: true
+      prefix: true,
+      primary: true
     },
     {
       name: "route_id",
       type: "text",
-      index: true,
-      prefix: true
+      prefix: true,
+      primary: true
     },
     {
       name: "trip_id",
       type: "text",
-      index: true,
-      prefix: true
+      prefix: true,
+      primary: true
     },
     {
       name: "stop_id",
       type: "text",
-      index: true,
-      prefix: true
+      prefix: true,
+      primary: true
     },
     {
       name: "stop_sequence",
       type: "integer",
       min: 0,
-      index: true
+      primary: true
     },
     {
       name: "show_on_stoptime",
@@ -2506,7 +2524,7 @@ var tripUpdates = {
   extension: "gtfs-realtime",
   schema: [
     {
-      name: "update_id",
+      name: "id",
       type: "text",
       required: true,
       primary: true,
@@ -2670,7 +2688,7 @@ var vehiclePositions = {
   extension: "gtfs-realtime",
   schema: [
     {
-      name: "update_id",
+      name: "id",
       type: "text",
       required: true,
       primary: true,
@@ -2909,17 +2927,12 @@ var deadheadTimes = {
   nonstandard: true,
   extension: "ods",
   schema: [
-    {
-      name: "id",
-      type: "integer",
-      primary: true,
-      prefix: true
-    },
     {
       name: "deadhead_id",
       type: "text",
       required: true,
       index: true,
+      primary: true,
       prefix: true
     },
     {
@@ -2956,6 +2969,7 @@ var deadheadTimes = {
       name: "location_sequence",
       type: "integer",
       required: true,
+      primary: true,
       min: 0,
       index: true
     },
@@ -3305,7 +3319,8 @@ function setDefaultConfig(initialConfig) {
     sqlitePath: ":memory:",
     ignoreDuplicates: false,
     ignoreErrors: false,
-    gtfsRealtimeExpirationSeconds: 0
+    gtfsRealtimeExpirationSeconds: 0,
+    verbose: true
   };
   return {
     ...defaults,
@@ -3654,18 +3669,31 @@ var createGtfsTables = (db) => {
       return;
     }
     const columns = model.schema.map((column) => {
-      let check = "";
+      const checks = [];
       if (column.min !== void 0 && column.max) {
-        check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
+        checks.push(
+          `${column.name} >= ${column.min} AND ${column.name} <= ${column.max}`
+        );
       } else if (column.min) {
-        check = `CHECK( ${column.name} >= ${column.min} )`;
+        checks.push(`${column.name} >= ${column.min}`);
       } else if (column.max) {
-        check = `CHECK( ${column.name} <= ${column.max} )`;
+        checks.push(`${column.name} <= ${column.max}`);
+      }
+      if (column.type === "integer") {
+        checks.push(
+          `(TYPEOF(${column.name}) = 'integer' OR ${column.name} IS NULL)`
+        );
+      }
+      if (column.type === "real") {
+        checks.push(
+          `(TYPEOF(${column.name}) = 'real' OR ${column.name} IS NULL)`
+        );
       }
       const required = column.required ? "NOT NULL" : "";
       const columnDefault = column.default ? "DEFAULT " + column.default : "";
       const columnCollation = column.nocase ? "COLLATE NOCASE" : "";
-      return `${column.name} ${column.type} ${check} ${required} ${columnDefault} ${columnCollation}`;
+      const checkClause = checks.length > 0 ? `CHECK(${checks.join(" AND ")})` : "";
+      return `${column.name} ${column.type} ${checkClause} ${required} ${columnDefault} ${columnCollation}`;
     });
     const primaryColumns = model.schema.filter((column) => column.primary);
     if (primaryColumns.length > 0) {
@@ -3708,38 +3736,15 @@ var formatGtfsLine = (line, model, totalLineCount) => {
       }
       continue;
     }
-    switch (type) {
-      case "date":
-        value = value.replace(/-/g, "");
-        if (value.length !== 8) {
-          throw new Error(
-            `Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
-          );
-        }
-        value = parseInt(value, 10);
-        break;
-      case "integer":
-        value = parseInt(value, 10);
-        break;
-      case "real":
-        value = parseFloat(value);
-        break;
-    }
-    if (Number.isNaN(value)) {
-      formattedLine[name] = null;
-      continue;
+    if (type === "date") {
+      value = value.replace(/-/g, "");
+      if (value.length !== 8) {
+        throw new Error(
+          `Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
+        );
+      }
     }
     formattedLine[name] = value;
-    if (min !== void 0 && value < min) {
-      throw new Error(
-        `Invalid value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}: below minimum value of ${min}.`
-      );
-    }
-    if (max !== void 0 && value > max) {
-      throw new Error(
-        `Invalid value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}: above maximum value of ${max}.`
-      );
-    }
   }
   for (const [timeColumnName, timestampColumnName] of TIME_COLUMN_PAIRS) {
     const value = formattedLine[timeColumnName];
@@ -3775,12 +3780,11 @@ var importGtfsFiles = (db, task) => mapSeries2(
       return;
     }
     task.log(`Importing - ${filename}\r`);
-    const columns = model.schema.filter((column) => column.name !== "id");
-    const placeholder = columns.map(({ name }) => `@${name}`).join(", ");
+    const placeholder = model.schema.map(({ name }) => `@${name}`).join(", ");
     const prefixedColumns = new Set(
-      columns.filter((col) => col.prefix).map((col) => col.name)
+      model.schema.filter((column) => column.prefix).map((column) => column.name)
     );
-    const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map((column) => column.name).join(", ")}) VALUES (${placeholder})`;
+    const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${model.schema.map((column) => column.name).join(", ")}) VALUES (${placeholder})`;
     const insert = db.prepare(prepareStatement);
     const insertLines = db.transaction((lines) => {
       for (const [rowNumber, line] of Object.entries(lines)) {
@@ -3808,7 +3812,7 @@ var importGtfsFiles = (db, task) => mapSeries2(
             );
           }
           task.logWarning(
-            `Check ${filename} for invalid data on row ${rowNumber}.`
+            `Check ${filename} for invalid data on line ${rowNumber + 1}.`
           );
           throw error;
         }