gtfs 4.19.0 → 4.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1196 +1,27 @@
1
1
  #!/usr/bin/env node
2
-
3
- // src/bin/gtfsrealtime-update.ts
2
+ import { Tt as formatError, lt as updateGtfsRealtime, xt as getConfig } from "../src-DuXcyA_N.js";
4
3
  import yargs from "yargs";
5
4
  import { hideBin } from "yargs/helpers";
6
5
  import PrettyError from "pretty-error";
7
6
 
8
- // src/lib/file-utils.ts
9
- import path from "path";
10
- import { existsSync } from "fs";
11
- import { homedir } from "os";
12
- import { mkdir, readFile, rm } from "fs/promises";
13
- import { omit, snakeCase } from "lodash-es";
14
- import sanitize from "sanitize-filename";
15
- import StreamZip from "node-stream-zip";
16
-
17
- // src/lib/log-utils.ts
18
- import { clearLine, cursorTo } from "readline";
19
- import { noop } from "lodash-es";
20
- import * as colors from "yoctocolors";
21
- function log(config) {
22
- if (config.verbose === false) {
23
- return noop;
24
- }
25
- if (config.logFunction) {
26
- return config.logFunction;
27
- }
28
- return (text, overwrite = false) => {
29
- if (overwrite && process.stdout.isTTY) {
30
- clearLine(process.stdout, 0);
31
- cursorTo(process.stdout, 0);
32
- } else {
33
- process.stdout.write("\n");
34
- }
35
- process.stdout.write(text);
36
- };
37
- }
38
- function logWarning(config) {
39
- if (config.logFunction) {
40
- return config.logFunction;
41
- }
42
- return (text) => {
43
- process.stdout.write(`
44
- ${formatWarning(text)}
45
- `);
46
- };
47
- }
48
- function logError(config) {
49
- if (config.logFunction) {
50
- return config.logFunction;
51
- }
52
- return (text) => {
53
- process.stdout.write(`
54
- ${formatError(text)}
55
- `);
56
- };
57
- }
58
- function formatWarning(text) {
59
- return colors.yellow(`${colors.underline("Warning")}: ${text}`);
60
- }
61
- function formatError(error) {
62
- const messageText = error instanceof Error ? error.message : error;
63
- const cleanMessage = messageText.replace(/^Error:\s*/i, "");
64
- return colors.red(`${colors.underline("Error")}: ${cleanMessage}`);
65
- }
66
-
67
- // src/lib/file-utils.ts
68
- var homeDirectory = homedir();
69
- async function getConfig(argv2) {
70
- let config;
71
- let data;
72
- try {
73
- if (argv2.configPath) {
74
- const configPath = path.resolve(untildify(argv2.configPath));
75
- data = await readFile(configPath, "utf8");
76
- config = Object.assign(JSON.parse(data), argv2);
77
- } else if (argv2.gtfsPath || argv2.gtfsUrl || argv2.sqlitePath) {
78
- const agencies = [
79
- ...argv2.gtfsPath ? [{ path: argv2.gtfsPath }] : [],
80
- ...argv2.gtfsUrl ? [{ url: argv2.gtfsUrl }] : []
81
- ];
82
- config = {
83
- agencies,
84
- ...omit(argv2, ["path", "url"])
85
- };
86
- } else if (existsSync(path.resolve("./config.json"))) {
87
- data = await readFile(path.resolve("./config.json"), "utf8");
88
- config = Object.assign(JSON.parse(data), argv2);
89
- log(config)("Using configuration from ./config.json");
90
- } else {
91
- throw new Error(
92
- "Cannot find configuration file. Use config-sample.json as a starting point, pass --configPath option."
93
- );
94
- }
95
- return config;
96
- } catch (error) {
97
- if (error instanceof SyntaxError) {
98
- throw new Error(
99
- `Cannot parse configuration file. Check to ensure that it is valid JSON. Error: ${error.message}`,
100
- { cause: error }
101
- );
102
- }
103
- throw error;
104
- }
105
- }
106
- function untildify(pathWithTilde) {
107
- return homeDirectory ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory) : pathWithTilde;
108
- }
109
-
110
- // src/lib/import-gtfs.ts
111
- import path2 from "path";
112
- import { createReadStream, existsSync as existsSync2, lstatSync } from "fs";
113
- import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "fs/promises";
114
- import { parse } from "csv-parse";
115
- import stripBomStream from "strip-bom-stream";
116
- import { temporaryDirectory } from "tempy";
117
- import mapSeries2 from "promise-map-series";
118
-
119
- // src/models/gtfs-realtime/trip-updates.ts
120
- var tripUpdates = {
121
- filenameBase: "trip_updates",
122
- extension: "gtfs-realtime",
123
- schema: [
124
- {
125
- name: "id",
126
- type: "text",
127
- required: true,
128
- primary: true,
129
- index: true,
130
- source: "id",
131
- prefix: true
132
- },
133
- {
134
- name: "vehicle_id",
135
- type: "text",
136
- index: true,
137
- source: "tripUpdate.vehicle.id",
138
- default: null,
139
- prefix: true
140
- },
141
- {
142
- name: "trip_id",
143
- type: "text",
144
- index: true,
145
- source: "tripUpdate.trip.tripId",
146
- default: null,
147
- prefix: true
148
- },
149
- {
150
- name: "trip_start_time",
151
- type: "text",
152
- source: "tripUpdate.trip.startTime",
153
- default: null
154
- },
155
- {
156
- name: "direction_id",
157
- type: "integer",
158
- source: "tripUpdate.trip.directionId",
159
- default: null
160
- },
161
- {
162
- name: "route_id",
163
- type: "text",
164
- index: true,
165
- source: "tripUpdate.trip.routeId",
166
- default: null,
167
- prefix: true
168
- },
169
- {
170
- name: "start_date",
171
- type: "text",
172
- source: "tripUpdate.trip.startDate",
173
- default: null
174
- },
175
- {
176
- name: "timestamp",
177
- type: "text",
178
- source: "tripUpdate.timestamp",
179
- default: null
180
- },
181
- {
182
- name: "schedule_relationship",
183
- type: "text",
184
- source: "tripUpdate.trip.scheduleRelationship",
185
- default: null
186
- },
187
- {
188
- name: "created_timestamp",
189
- type: "integer",
190
- required: true
191
- },
192
- {
193
- name: "expiration_timestamp",
194
- type: "integer",
195
- required: true
196
- }
197
- ]
198
- };
199
-
200
- // src/models/gtfs-realtime/stop-time-updates.ts
201
- var stopTimeUpdates = {
202
- filenameBase: "stop_time_updates",
203
- extension: "gtfs-realtime",
204
- schema: [
205
- {
206
- name: "trip_id",
207
- type: "text",
208
- index: true,
209
- source: "parent.tripUpdate.trip.tripId",
210
- default: null,
211
- prefix: true
212
- },
213
- {
214
- name: "trip_start_time",
215
- type: "text",
216
- source: "parent.tripUpdate.trip.startTime",
217
- default: null
218
- },
219
- {
220
- name: "direction_id",
221
- type: "integer",
222
- source: "parent.tripUpdate.trip.directionId",
223
- default: null
224
- },
225
- {
226
- name: "route_id",
227
- type: "text",
228
- index: true,
229
- source: "parent.tripUpdate.trip.routeId",
230
- default: null,
231
- prefix: true
232
- },
233
- {
234
- name: "stop_id",
235
- type: "text",
236
- index: true,
237
- source: "stopId",
238
- default: null,
239
- prefix: true
240
- },
241
- {
242
- name: "stop_sequence",
243
- type: "integer",
244
- source: "stopSequence",
245
- default: null
246
- },
247
- {
248
- name: "arrival_delay",
249
- type: "integer",
250
- source: "arrival.delay",
251
- default: null
252
- },
253
- {
254
- name: "departure_delay",
255
- type: "integer",
256
- source: "departure.delay",
257
- default: null
258
- },
259
- {
260
- name: "departure_timestamp",
261
- type: "text",
262
- source: "departure.time",
263
- default: null
264
- },
265
- {
266
- name: "arrival_timestamp",
267
- type: "text",
268
- source: "arrival.time",
269
- default: null
270
- },
271
- {
272
- name: "schedule_relationship",
273
- type: "text",
274
- source: "scheduleRelationship",
275
- default: null
276
- },
277
- {
278
- name: "created_timestamp",
279
- type: "integer",
280
- required: true
281
- },
282
- {
283
- name: "expiration_timestamp",
284
- type: "integer",
285
- required: true
286
- }
287
- ]
288
- };
289
-
290
- // src/models/gtfs-realtime/vehicle-positions.ts
291
- var vehiclePositions = {
292
- filenameBase: "vehicle_positions",
293
- extension: "gtfs-realtime",
294
- schema: [
295
- {
296
- name: "id",
297
- type: "text",
298
- required: true,
299
- primary: true,
300
- index: true,
301
- source: "id",
302
- prefix: true
303
- },
304
- {
305
- name: "bearing",
306
- type: "real",
307
- source: "vehicle.position.bearing",
308
- default: null
309
- },
310
- {
311
- name: "latitude",
312
- type: "real",
313
- min: -90,
314
- max: 90,
315
- source: "vehicle.position.latitude",
316
- default: null
317
- },
318
- {
319
- name: "longitude",
320
- type: "real",
321
- source: "vehicle.position.longitude",
322
- min: -180,
323
- max: 180,
324
- default: null
325
- },
326
- {
327
- name: "speed",
328
- type: "real",
329
- min: 0,
330
- source: "vehicle.position.speed",
331
- default: null
332
- },
333
- {
334
- name: "current_stop_sequence",
335
- type: "integer",
336
- source: "vehicle.currentStopSequence",
337
- default: null
338
- },
339
- {
340
- name: "trip_id",
341
- type: "text",
342
- index: true,
343
- source: "vehicle.trip.tripId",
344
- default: null,
345
- prefix: true
346
- },
347
- {
348
- name: "trip_start_date",
349
- type: "text",
350
- index: true,
351
- source: "vehicle.trip.startDate",
352
- default: null
353
- },
354
- {
355
- name: "trip_start_time",
356
- type: "text",
357
- index: true,
358
- source: "vehicle.trip.startTime",
359
- default: null
360
- },
361
- {
362
- name: "congestion_level",
363
- type: "text",
364
- source: "vehicle.congestionLevel",
365
- default: null
366
- },
367
- {
368
- name: "occupancy_status",
369
- type: "text",
370
- source: "vehicle.occupancyStatus",
371
- default: null
372
- },
373
- {
374
- name: "occupancy_percentage",
375
- type: "integer",
376
- source: "vehicle.occupancyPercentage",
377
- default: null
378
- },
379
- {
380
- name: "vehicle_stop_status",
381
- type: "text",
382
- source: "vehicle.vehicleStopStatus",
383
- default: null
384
- },
385
- {
386
- name: "vehicle_id",
387
- type: "text",
388
- index: true,
389
- source: "vehicle.vehicle.id",
390
- default: null,
391
- prefix: true
392
- },
393
- {
394
- name: "vehicle_label",
395
- type: "text",
396
- source: "vehicle.vehicle.label",
397
- default: null
398
- },
399
- {
400
- name: "vehicle_license_plate",
401
- type: "text",
402
- source: "vehicle.vehicle.licensePlate",
403
- default: null
404
- },
405
- {
406
- name: "vehicle_wheelchair_accessible",
407
- type: "text",
408
- source: "vehicle.vehicle.wheelchairAccessible",
409
- default: null
410
- },
411
- {
412
- name: "timestamp",
413
- type: "text",
414
- source: "vehicle.timestamp",
415
- default: null
416
- },
417
- {
418
- name: "created_timestamp",
419
- type: "integer",
420
- required: true
421
- },
422
- {
423
- name: "expiration_timestamp",
424
- type: "integer",
425
- required: true
426
- }
427
- ]
428
- };
429
-
430
- // src/models/gtfs-realtime/service-alerts.ts
431
- var serviceAlerts = {
432
- filenameBase: "service_alerts",
433
- extension: "gtfs-realtime",
434
- schema: [
435
- {
436
- name: "id",
437
- type: "text",
438
- required: true,
439
- primary: true,
440
- index: true,
441
- source: "id",
442
- prefix: true
443
- },
444
- {
445
- name: "active_period",
446
- type: "json",
447
- source: "alert.activePeriod"
448
- },
449
- {
450
- name: "cause",
451
- type: "text",
452
- source: "alert.cause"
453
- },
454
- {
455
- name: "effect",
456
- type: "text",
457
- source: "alert.effect"
458
- },
459
- {
460
- name: "url",
461
- type: "text",
462
- source: "alert.url.translation[0].text",
463
- default: ""
464
- },
465
- {
466
- name: "start_time",
467
- type: "text",
468
- source: "alert.activePeriod[0].start",
469
- default: null
470
- },
471
- {
472
- name: "end_time",
473
- type: "text",
474
- source: "alert.activePeriod[0].end",
475
- default: null
476
- },
477
- {
478
- name: "header_text",
479
- type: "text",
480
- required: true,
481
- source: "alert.headerText.translation[0].text",
482
- default: ""
483
- },
484
- {
485
- name: "description_text",
486
- type: "text",
487
- required: true,
488
- source: "alert.descriptionText.translation[0].text",
489
- default: ""
490
- },
491
- {
492
- name: "tts_header_text",
493
- type: "text",
494
- source: "alert.ttsHeaderText.translation[0].text"
495
- },
496
- {
497
- name: "tts_description_text",
498
- type: "text",
499
- source: "alert.ttsDescriptionText.translation[0].text"
500
- },
501
- {
502
- name: "severity_level",
503
- type: "text",
504
- source: "alert.severityLevel"
505
- },
506
- {
507
- name: "created_timestamp",
508
- type: "integer",
509
- required: true
510
- },
511
- {
512
- name: "expiration_timestamp",
513
- type: "integer",
514
- required: true
515
- }
516
- ]
517
- };
518
-
519
- // src/models/gtfs-realtime/service-alert-informed_entities.ts
520
- var serviceAlertInformedEntities = {
521
- filenameBase: "service_alert_informed_entities",
522
- extension: "gtfs-realtime",
523
- schema: [
524
- {
525
- name: "alert_id",
526
- type: "text",
527
- required: true,
528
- primary: true,
529
- source: "parent.id",
530
- prefix: true
531
- },
532
- {
533
- name: "stop_id",
534
- type: "text",
535
- primary: true,
536
- source: "stopId",
537
- default: null,
538
- prefix: true
539
- },
540
- {
541
- name: "route_id",
542
- type: "text",
543
- primary: true,
544
- source: "routeId",
545
- default: null,
546
- prefix: true
547
- },
548
- {
549
- name: "route_type",
550
- type: "integer",
551
- primary: true,
552
- source: "routeType",
553
- default: null
554
- },
555
- {
556
- name: "trip_id",
557
- type: "text",
558
- primary: true,
559
- source: "trip.tripId",
560
- default: null,
561
- prefix: true
562
- },
563
- {
564
- name: "direction_id",
565
- type: "integer",
566
- primary: true,
567
- source: "directionId",
568
- default: null
569
- },
570
- {
571
- name: "created_timestamp",
572
- type: "integer",
573
- required: true
574
- },
575
- {
576
- name: "expiration_timestamp",
577
- type: "integer",
578
- required: true
579
- }
580
- ]
581
- };
582
-
583
- // src/lib/db.ts
584
- import Database from "better-sqlite3";
585
-
586
- // src/lib/errors.ts
587
- var GtfsError = class extends Error {
588
- code;
589
- category;
590
- isOperational;
591
- statusCode;
592
- details;
593
- constructor(message, options) {
594
- super(message, { cause: options.cause });
595
- this.name = "GtfsError";
596
- this.code = options.code;
597
- this.category = options.category;
598
- this.isOperational = options.isOperational ?? true;
599
- this.statusCode = options.statusCode;
600
- this.details = options.details;
601
- }
602
- };
603
- function isGtfsError(error) {
604
- if (!error || typeof error !== "object") {
605
- return false;
606
- }
607
- const candidate = error;
608
- return candidate.name === "GtfsError" && typeof candidate.message === "string" && typeof candidate.code === "string" && typeof candidate.category === "string" && typeof candidate.isOperational === "boolean";
609
- }
610
- function toGtfsError(error, fallback) {
611
- if (isGtfsError(error)) {
612
- return error;
613
- }
614
- return new GtfsError(fallback.message, {
615
- ...fallback,
616
- cause: error
617
- });
618
- }
619
- function addImportError(report, error) {
620
- report.errors.push(error);
621
- report.errorCountsByCode[error.code] = (report.errorCountsByCode[error.code] ?? 0) + 1;
622
- }
623
- function formatGtfsError(error, options = { verbosity: "developer" }) {
624
- if (!isGtfsError(error)) {
625
- const message = error instanceof Error ? error.message : String(error);
626
- return options.verbosity === "user" ? message : `UNKNOWN_ERROR: ${message}`;
627
- }
628
- if (options.verbosity === "user") {
629
- return error.message;
630
- }
631
- return [
632
- `${error.code}: ${error.message}`,
633
- `category=${error.category}`,
634
- error.statusCode !== void 0 ? `statusCode=${error.statusCode}` : null,
635
- error.details ? `details=${JSON.stringify(error.details)}` : null
636
- ].filter(Boolean).join(" | ");
637
- }
638
-
639
- // src/lib/db.ts
640
- var dbs = {};
641
- function setupDb(sqlitePath) {
642
- const db = new Database(untildify(sqlitePath));
643
- db.pragma("journal_mode = OFF");
644
- db.pragma("synchronous = OFF");
645
- db.pragma("temp_store = MEMORY");
646
- dbs[sqlitePath] = db;
647
- return db;
648
- }
649
- function openDb(config = null) {
650
- if (config) {
651
- const { sqlitePath = ":memory:", db } = config;
652
- if (db) {
653
- return db;
654
- }
655
- if (dbs[sqlitePath]) {
656
- return dbs[sqlitePath];
657
- }
658
- return setupDb(sqlitePath);
659
- }
660
- if (Object.keys(dbs).length === 0) {
661
- return setupDb(":memory:");
662
- }
663
- if (Object.keys(dbs).length === 1) {
664
- const filename = Object.keys(dbs)[0];
665
- return dbs[filename];
666
- }
667
- if (Object.keys(dbs).length > 1) {
668
- throw new GtfsError(
669
- "Multiple databases open, please specify which one to use.",
670
- {
671
- code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
672
- category: "database" /* DATABASE */,
673
- details: { openDatabaseCount: Object.keys(dbs).length }
674
- }
675
- );
676
- }
677
- throw new GtfsError("Unable to find database connection.", {
678
- code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
679
- category: "database" /* DATABASE */
680
- });
681
- }
682
-
683
- // src/lib/geojson-utils.ts
684
- import {
685
- cloneDeep,
686
- compact,
687
- filter,
688
- groupBy,
689
- last,
690
- omit as omit2,
691
- sortBy,
692
- omitBy
693
- } from "lodash-es";
694
- import { feature, featureCollection } from "@turf/helpers";
695
-
696
- // src/lib/import-gtfs-realtime.ts
697
- import GtfsRealtimeBindings from "gtfs-realtime-bindings";
698
- import mapSeries from "promise-map-series";
699
- import { get } from "lodash-es";
700
-
701
- // src/lib/utils.ts
702
- import sqlString from "sqlstring-sqlite";
703
- import Long from "long";
704
- function validateConfigForImport(config) {
705
- if (!config.agencies || config.agencies.length === 0) {
706
- throw new GtfsError("No `agencies` specified in config", {
707
- code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
708
- category: "config" /* CONFIG */,
709
- details: { field: "agencies" }
710
- });
711
- }
712
- for (const [index, agency2] of config.agencies.entries()) {
713
- if (!agency2.path && !agency2.url) {
714
- throw new GtfsError(
715
- `No Agency \`url\` or \`path\` specified in config for agency index ${index}.`,
716
- {
717
- code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
718
- category: "config" /* CONFIG */,
719
- details: { agencyIndex: index }
720
- }
721
- );
722
- }
723
- }
724
- return config;
725
- }
726
- function setDefaultConfig(initialConfig) {
727
- const defaults = {
728
- sqlitePath: ":memory:",
729
- ignoreDuplicates: false,
730
- ignoreErrors: false,
731
- gtfsRealtimeExpirationSeconds: 0,
732
- verbose: true,
733
- downloadTimeout: 3e4
734
- };
735
- return {
736
- ...defaults,
737
- ...initialConfig
738
- };
739
- }
740
- function convertLongTimeToDate(longDate) {
741
- const { high, low, unsigned } = longDate;
742
- return new Date(
743
- Long.fromBits(low, high, unsigned).toNumber() * 1e3
744
- ).toISOString();
745
- }
746
- function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
747
- if (!columnShouldBePrefixed || prefix === void 0 || value === null || value === void 0) {
748
- return value;
749
- }
750
- return `${prefix}${value}`;
751
- }
752
- function pluralize(singularWord, pluralWord, count) {
753
- return count === 1 ? singularWord : pluralWord;
754
- }
755
-
756
- // src/lib/import-gtfs-realtime.ts
757
- var BATCH_SIZE = 1e3;
758
- var MAX_RETRIES = 3;
759
- var RETRY_DELAY = 1e3;
760
- function prepareRealtimeFieldValue(entity, column, task) {
761
- if (column.name === "created_timestamp") {
762
- return task.currentTimestamp;
763
- }
764
- if (column.name === "expiration_timestamp") {
765
- return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
766
- }
767
- const baseValue = column.source === void 0 ? column.default : get(entity, column.source, column.default);
768
- const timeAdjustedValue = baseValue?.__isLong__ ? convertLongTimeToDate(baseValue) : baseValue;
769
- const prefixedValue = applyPrefixToValue(
770
- timeAdjustedValue,
771
- column.prefix,
772
- task.prefix
773
- );
774
- return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
775
- }
776
- function createPreparedStatement(db, model) {
777
- const columns = model.schema.map((column) => column.name);
778
- const placeholders = model.schema.map(() => "?").join(", ");
779
- return db.prepare(
780
- `REPLACE INTO ${model.filenameBase} (${columns.join(", ")}) VALUES (${placeholders})`
781
- );
782
- }
783
- async function processBatch(items, batchSize, processor) {
784
- let totalRecordCount = 0;
785
- let totalErrorCount = 0;
786
- for (let i = 0; i < items.length; i += batchSize) {
787
- const batch = items.slice(i, i + batchSize);
788
- try {
789
- const result = await processor(batch);
790
- totalRecordCount += result.recordCount;
791
- totalErrorCount += result.errorCount;
792
- } catch (error) {
793
- const errorMessage = error instanceof Error ? error.message : String(error);
794
- totalErrorCount += batch.length;
795
- console.error(`Batch processing error: ${errorMessage}`);
796
- }
797
- }
798
- return { recordCount: totalRecordCount, errorCount: totalErrorCount };
799
- }
800
- async function fetchGtfsRealtimeData(type, task) {
801
- const urlConfig = getUrlConfig(type, task);
802
- if (!urlConfig) {
803
- return null;
804
- }
805
- task.log(`Importing - GTFS-Realtime from ${urlConfig.url}`);
806
- for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
807
- try {
808
- const response = await fetch(urlConfig.url, {
809
- method: "GET",
810
- redirect: "follow",
811
- headers: {
812
- "User-Agent": "node-gtfs",
813
- ...urlConfig.headers ?? {},
814
- "Accept-Encoding": "gzip"
815
- },
816
- signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
817
- });
818
- if (!response.ok) {
819
- throw new GtfsError(`HTTP ${response.status}: ${response.statusText}`, {
820
- code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
821
- category: "download" /* DOWNLOAD */,
822
- statusCode: response.status,
823
- details: {
824
- url: urlConfig.url,
825
- status: response.status,
826
- statusText: response.statusText
827
- }
828
- });
829
- }
830
- const buffer = await response.arrayBuffer();
831
- const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
832
- new Uint8Array(buffer)
833
- );
834
- const feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
835
- enums: String,
836
- longs: String,
837
- bytes: String,
838
- defaults: false,
839
- arrays: true,
840
- objects: true,
841
- oneofs: true
842
- });
843
- return feedMessage;
844
- } catch (error) {
845
- const gtfsError = toGtfsError(error, {
846
- message: error instanceof Error ? error.message : String(error),
847
- code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
848
- category: "download" /* DOWNLOAD */,
849
- details: { type, url: urlConfig.url }
850
- });
851
- if (attempt === MAX_RETRIES) {
852
- if (task.ignoreErrors) {
853
- task.logError(
854
- `Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${gtfsError.message}`
855
- );
856
- if (task.report) {
857
- addImportError(task.report, gtfsError);
858
- }
859
- return null;
860
- }
861
- throw gtfsError;
862
- }
863
- task.logWarning(
864
- `Attempt ${attempt} failed for ${type}: ${gtfsError.message}`
865
- );
866
- await new Promise(
867
- (resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
868
- );
869
- }
870
- }
871
- return null;
872
- }
873
- function getUrlConfig(type, task) {
874
- switch (type) {
875
- case "alerts":
876
- return task.realtimeAlerts;
877
- case "tripupdates":
878
- return task.realtimeTripUpdates;
879
- case "vehiclepositions":
880
- return task.realtimeVehiclePositions;
881
- default:
882
- return void 0;
883
- }
884
- }
885
- function createServiceAlertsProcessor(db, task) {
886
- const alertStmt = createPreparedStatement(db, serviceAlerts);
887
- const informedEntityStmt = createPreparedStatement(
888
- db,
889
- serviceAlertInformedEntities
890
- );
891
- const deleteInformedEntitiesStmt = db.prepare(
892
- `DELETE FROM ${serviceAlertInformedEntities.filenameBase} WHERE alert_id = ?`
893
- );
894
- return async (batch) => {
895
- let recordCount = 0;
896
- let errorCount = 0;
897
- db.transaction(() => {
898
- for (const entity of batch) {
899
- try {
900
- const alertId = applyPrefixToValue(entity.id, true, task.prefix);
901
- deleteInformedEntitiesStmt.run(alertId);
902
- const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
903
- alertStmt.run(alertValues);
904
- recordCount++;
905
- if (entity.alert?.informedEntity?.length) {
906
- for (const informedEntity of entity.alert.informedEntity) {
907
- informedEntity.parent = entity;
908
- const entityValues = serviceAlertInformedEntities.schema.map(
909
- (column) => prepareRealtimeFieldValue(informedEntity, column, task)
910
- );
911
- informedEntityStmt.run(entityValues);
912
- recordCount++;
913
- }
914
- }
915
- } catch (error) {
916
- const errorMessage = error instanceof Error ? error.message : String(error);
917
- errorCount++;
918
- task.logWarning(`Alert processing error: ${errorMessage}`);
919
- }
920
- }
921
- })();
922
- return { recordCount, errorCount };
923
- };
924
- }
925
- function createTripUpdatesProcessor(db, task) {
926
- const tripUpdateStmt = createPreparedStatement(
927
- db,
928
- tripUpdates
929
- );
930
- const stopTimeStmt = createPreparedStatement(
931
- db,
932
- stopTimeUpdates
933
- );
934
- const deleteStopTimesByTripStmt = db.prepare(
935
- `DELETE FROM ${stopTimeUpdates.filenameBase} WHERE trip_id = ? AND trip_start_time IS ?`
936
- );
937
- return async (batch) => {
938
- let recordCount = 0;
939
- let errorCount = 0;
940
- db.transaction(() => {
941
- for (const entity of batch) {
942
- try {
943
- const tripUpdateValues = tripUpdates.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
944
- tripUpdateStmt.run(tripUpdateValues);
945
- recordCount++;
946
- if (entity.tripUpdate?.stopTimeUpdate?.length) {
947
- const tripId = applyPrefixToValue(
948
- entity.tripUpdate?.trip?.tripId ?? null,
949
- true,
950
- task.prefix
951
- );
952
- const tripStartTime = entity.tripUpdate?.trip?.startTime ?? null;
953
- if (tripId !== null) {
954
- deleteStopTimesByTripStmt.run(tripId, tripStartTime);
955
- }
956
- for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
957
- stopTimeUpdate.parent = entity;
958
- const stopTimeValues = stopTimeUpdates.schema.map(
959
- (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
960
- );
961
- stopTimeStmt.run(stopTimeValues);
962
- recordCount++;
963
- }
964
- }
965
- } catch (error) {
966
- const errorMessage = error instanceof Error ? error.message : String(error);
967
- errorCount++;
968
- task.logWarning(`Trip update processing error: ${errorMessage}`);
969
- }
970
- }
971
- })();
972
- return { recordCount, errorCount };
973
- };
974
- }
975
- function createVehiclePositionsProcessor(db, task) {
976
- const vehiclePositionStmt = createPreparedStatement(
977
- db,
978
- vehiclePositions
979
- );
980
- return async (batch) => {
981
- let recordCount = 0;
982
- let errorCount = 0;
983
- db.transaction(() => {
984
- for (const entity of batch) {
985
- try {
986
- const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
987
- vehiclePositionStmt.run(fieldValues);
988
- recordCount++;
989
- } catch (error) {
990
- const errorMessage = error instanceof Error ? error.message : String(error);
991
- errorCount++;
992
- task.logWarning(`Vehicle position processing error: ${errorMessage}`);
993
- }
994
- }
995
- })();
996
- return { recordCount, errorCount };
997
- };
998
- }
999
- function removeExpiredRealtimeData(config) {
1000
- const db = openDb(config);
1001
- log(config)(`Removing expired GTFS-Realtime data`);
1002
- db.transaction(() => {
1003
- const tables = [
1004
- "vehicle_positions",
1005
- "trip_updates",
1006
- "stop_time_updates",
1007
- "service_alerts",
1008
- "service_alert_informed_entities"
1009
- ];
1010
- for (const table of tables) {
1011
- db.prepare(
1012
- `DELETE FROM ${table} WHERE expiration_timestamp <= strftime('%s','now')`
1013
- ).run();
1014
- }
1015
- })();
1016
- log(config)(`Removed expired GTFS-Realtime data\r`, true);
1017
- }
1018
- async function updateGtfsRealtimeData(task) {
1019
- if (!task.realtimeAlerts && !task.realtimeTripUpdates && !task.realtimeVehiclePositions) {
1020
- return;
1021
- }
1022
- const [alertsData, tripUpdatesData, vehiclePositionsData] = await Promise.all(
1023
- [
1024
- task.realtimeAlerts?.url ? fetchGtfsRealtimeData("alerts", task) : null,
1025
- task.realtimeTripUpdates?.url ? fetchGtfsRealtimeData("tripupdates", task) : null,
1026
- task.realtimeVehiclePositions?.url ? fetchGtfsRealtimeData("vehiclepositions", task) : null
1027
- ]
1028
- );
1029
- const db = openDb({ sqlitePath: task.sqlitePath });
1030
- const recordCounts = {
1031
- alerts: 0,
1032
- tripupdates: 0,
1033
- vehiclepositions: 0
1034
- };
1035
- if (alertsData?.entity?.length) {
1036
- const result = await processBatch(
1037
- alertsData.entity,
1038
- BATCH_SIZE,
1039
- createServiceAlertsProcessor(db, task)
1040
- );
1041
- recordCounts.alerts = result.recordCount;
1042
- }
1043
- if (tripUpdatesData?.entity?.length) {
1044
- const result = await processBatch(
1045
- tripUpdatesData.entity,
1046
- BATCH_SIZE,
1047
- createTripUpdatesProcessor(db, task)
1048
- );
1049
- recordCounts.tripupdates = result.recordCount;
1050
- }
1051
- if (vehiclePositionsData?.entity?.length) {
1052
- const result = await processBatch(
1053
- vehiclePositionsData.entity,
1054
- BATCH_SIZE,
1055
- createVehiclePositionsProcessor(db, task)
1056
- );
1057
- recordCounts.vehiclepositions = result.recordCount;
1058
- }
1059
- task.log(
1060
- `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
1061
- );
1062
- }
1063
- async function updateGtfsRealtime(initialConfig) {
1064
- const config = setDefaultConfig(initialConfig);
1065
- validateConfigForImport(config);
1066
- try {
1067
- openDb(config);
1068
- const agencyCount = config.agencies.length;
1069
- log(config)(
1070
- `Starting GTFS-Realtime refresh for ${pluralize(
1071
- "agency",
1072
- "agencies",
1073
- agencyCount
1074
- )} using SQLite database at ${config.sqlitePath}`
1075
- );
1076
- removeExpiredRealtimeData(config);
1077
- await mapSeries(config.agencies, async (agency2) => {
1078
- let task;
1079
- try {
1080
- task = {
1081
- realtimeAlerts: agency2.realtimeAlerts,
1082
- realtimeTripUpdates: agency2.realtimeTripUpdates,
1083
- realtimeVehiclePositions: agency2.realtimeVehiclePositions,
1084
- downloadTimeout: config.downloadTimeout,
1085
- gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
1086
- ignoreErrors: config.ignoreErrors,
1087
- sqlitePath: config.sqlitePath,
1088
- prefix: agency2.prefix,
1089
- currentTimestamp: Math.floor(Date.now() / 1e3),
1090
- log: log(config),
1091
- logWarning: logWarning(config),
1092
- logError: logError(config)
1093
- };
1094
- await updateGtfsRealtimeData(task);
1095
- } catch (error) {
1096
- const gtfsError = toGtfsError(error, {
1097
- message: error instanceof Error ? error.message : String(error),
1098
- code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
1099
- category: "database" /* DATABASE */,
1100
- details: { sqlitePath: task?.sqlitePath ?? config.sqlitePath }
1101
- });
1102
- if (config.ignoreErrors) {
1103
- logError(config)(formatGtfsError(gtfsError));
1104
- if (task?.report) {
1105
- addImportError(task.report, gtfsError);
1106
- }
1107
- } else {
1108
- throw gtfsError;
1109
- }
1110
- }
1111
- });
1112
- log(config)(
1113
- `Completed GTFS-Realtime refresh for ${pluralize(
1114
- "agency",
1115
- "agencies",
1116
- agencyCount
1117
- )}
1118
- `
1119
- );
1120
- } catch (error) {
1121
- if (error.code === "SQLITE_CANTOPEN") {
1122
- const dbOpenError = new GtfsError(
1123
- `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`,
1124
- {
1125
- code: "DB_OPEN_FAILED" /* DB_OPEN_FAILED */,
1126
- category: "database" /* DATABASE */,
1127
- details: {
1128
- sqlitePath: config.sqlitePath,
1129
- dbCode: error.code
1130
- },
1131
- cause: error
1132
- }
1133
- );
1134
- logError(config)(dbOpenError.message);
1135
- throw dbOpenError;
1136
- }
1137
- throw toGtfsError(error, {
1138
- message: error instanceof Error ? error.message : String(error),
1139
- code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
1140
- category: "database" /* DATABASE */
1141
- });
1142
- }
1143
- }
1144
-
1145
- // src/lib/export.ts
1146
- import path3 from "path";
1147
- import { writeFile as writeFile2 } from "fs/promises";
1148
- import { without, compact as compact2 } from "lodash-es";
1149
- import { stringify } from "csv-stringify";
1150
- import sqlString2 from "sqlstring-sqlite";
1151
- import mapSeries3 from "promise-map-series";
1152
-
1153
- // src/lib/advancedQuery.ts
1154
- import sqlString3 from "sqlstring-sqlite";
1155
-
1156
- // src/lib/gtfs/routes.ts
1157
- import { omit as omit3, pick } from "lodash-es";
1158
-
1159
- // src/lib/gtfs/shapes.ts
1160
- import { compact as compact3, omit as omit4, pick as pick2 } from "lodash-es";
1161
- import { featureCollection as featureCollection2 } from "@turf/helpers";
1162
-
1163
- // src/lib/gtfs/stops.ts
1164
- import { omit as omit5, orderBy, pick as pick3 } from "lodash-es";
1165
-
1166
- // src/lib/gtfs/stop-times.ts
1167
- import { omit as omit6 } from "lodash-es";
1168
- import sqlString4 from "sqlstring-sqlite";
1169
-
1170
- // src/lib/gtfs/trips.ts
1171
- import { omit as omit7 } from "lodash-es";
1172
- import sqlString5 from "sqlstring-sqlite";
1173
-
1174
- // src/bin/gtfsrealtime-update.ts
1175
- var pe = new PrettyError();
1176
- var argv = yargs(hideBin(process.argv)).usage("Usage: $0 --configPath ./config.json").help().option("c", {
1177
- alias: "configPath",
1178
- describe: "Path to config file",
1179
- type: "string"
7
+ //#region src/bin/gtfsrealtime-update.ts
8
+ const pe = new PrettyError();
9
+ const argv = yargs(hideBin(process.argv)).usage("Usage: $0 --configPath ./config.json").help().option("c", {
10
+ alias: "configPath",
11
+ describe: "Path to config file",
12
+ type: "string"
1180
13
  }).default("configPath", void 0).parseSync();
1181
- var handleError = (error = "Unknown Error") => {
1182
- process.stdout.write(`
1183
- ${formatError(error)}
1184
- `);
1185
- console.error(pe.render(error));
1186
- process.exit(1);
14
+ const handleError = (error = "Unknown Error") => {
15
+ process.stdout.write(`\n${formatError(error)}\n`);
16
+ console.error(pe.render(error));
17
+ process.exit(1);
1187
18
  };
1188
- var setupImport = async () => {
1189
- const config = await getConfig({
1190
- configPath: argv.configPath
1191
- });
1192
- await updateGtfsRealtime(config);
1193
- process.exit();
19
+ const setupImport = async () => {
20
+ await updateGtfsRealtime(await getConfig({ configPath: argv.configPath }));
21
+ process.exit();
1194
22
  };
1195
23
  setupImport().catch(handleError);
24
+
25
+ //#endregion
26
+ export { };
1196
27
  //# sourceMappingURL=gtfsrealtime-update.js.map