gtfs 4.18.3 → 4.18.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/gtfs-export.js +33 -4
- package/dist/bin/gtfs-export.js.map +1 -1
- package/dist/bin/gtfs-import.js +381 -74
- package/dist/bin/gtfs-import.js.map +1 -1
- package/dist/bin/gtfsrealtime-update.js +135 -19
- package/dist/bin/gtfsrealtime-update.js.map +1 -1
- package/dist/index.d.ts +73 -1
- package/dist/index.js +501 -92
- package/dist/index.js.map +1 -1
- package/package.json +15 -15
package/dist/bin/gtfs-import.js
CHANGED
|
@@ -101,7 +101,8 @@ async function getConfig(argv2) {
|
|
|
101
101
|
} catch (error) {
|
|
102
102
|
if (error instanceof SyntaxError) {
|
|
103
103
|
throw new Error(
|
|
104
|
-
`Cannot parse configuration file. Check to ensure that it is valid JSON. Error: ${error.message}
|
|
104
|
+
`Cannot parse configuration file. Check to ensure that it is valid JSON. Error: ${error.message}`,
|
|
105
|
+
{ cause: error }
|
|
105
106
|
);
|
|
106
107
|
}
|
|
107
108
|
throw error;
|
|
@@ -114,7 +115,8 @@ async function unzip(zipfilePath, exportPath) {
|
|
|
114
115
|
await zip.close();
|
|
115
116
|
} catch (error) {
|
|
116
117
|
throw new Error(
|
|
117
|
-
`Failed to extract zip file: ${error instanceof Error ? error.message : "Unknown error"}
|
|
118
|
+
`Failed to extract zip file: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
119
|
+
{ cause: error }
|
|
118
120
|
);
|
|
119
121
|
}
|
|
120
122
|
}
|
|
@@ -4108,6 +4110,73 @@ var vehicles = {
|
|
|
4108
4110
|
|
|
4109
4111
|
// src/lib/db.ts
|
|
4110
4112
|
import Database from "better-sqlite3";
|
|
4113
|
+
|
|
4114
|
+
// src/lib/errors.ts
|
|
4115
|
+
var GtfsError = class extends Error {
|
|
4116
|
+
code;
|
|
4117
|
+
category;
|
|
4118
|
+
isOperational;
|
|
4119
|
+
statusCode;
|
|
4120
|
+
details;
|
|
4121
|
+
constructor(message, options) {
|
|
4122
|
+
super(message, { cause: options.cause });
|
|
4123
|
+
this.name = "GtfsError";
|
|
4124
|
+
this.code = options.code;
|
|
4125
|
+
this.category = options.category;
|
|
4126
|
+
this.isOperational = options.isOperational ?? true;
|
|
4127
|
+
this.statusCode = options.statusCode;
|
|
4128
|
+
this.details = options.details;
|
|
4129
|
+
}
|
|
4130
|
+
};
|
|
4131
|
+
function isGtfsError(error) {
|
|
4132
|
+
if (!error || typeof error !== "object") {
|
|
4133
|
+
return false;
|
|
4134
|
+
}
|
|
4135
|
+
const candidate = error;
|
|
4136
|
+
return candidate.name === "GtfsError" && typeof candidate.message === "string" && typeof candidate.code === "string" && typeof candidate.category === "string" && typeof candidate.isOperational === "boolean";
|
|
4137
|
+
}
|
|
4138
|
+
function toGtfsError(error, fallback) {
|
|
4139
|
+
if (isGtfsError(error)) {
|
|
4140
|
+
return error;
|
|
4141
|
+
}
|
|
4142
|
+
return new GtfsError(fallback.message, {
|
|
4143
|
+
...fallback,
|
|
4144
|
+
cause: error
|
|
4145
|
+
});
|
|
4146
|
+
}
|
|
4147
|
+
function createImportReport() {
|
|
4148
|
+
return {
|
|
4149
|
+
errors: [],
|
|
4150
|
+
warnings: [],
|
|
4151
|
+
errorCountsByCode: {},
|
|
4152
|
+
warningCountsByCode: {}
|
|
4153
|
+
};
|
|
4154
|
+
}
|
|
4155
|
+
function addImportError(report, error) {
|
|
4156
|
+
report.errors.push(error);
|
|
4157
|
+
report.errorCountsByCode[error.code] = (report.errorCountsByCode[error.code] ?? 0) + 1;
|
|
4158
|
+
}
|
|
4159
|
+
function addImportWarning(report, warning) {
|
|
4160
|
+
report.warnings.push(warning);
|
|
4161
|
+
report.warningCountsByCode[warning.code] = (report.warningCountsByCode[warning.code] ?? 0) + 1;
|
|
4162
|
+
}
|
|
4163
|
+
function formatGtfsError(error, options = { verbosity: "developer" }) {
|
|
4164
|
+
if (!isGtfsError(error)) {
|
|
4165
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4166
|
+
return options.verbosity === "user" ? message : `UNKNOWN_ERROR: ${message}`;
|
|
4167
|
+
}
|
|
4168
|
+
if (options.verbosity === "user") {
|
|
4169
|
+
return error.message;
|
|
4170
|
+
}
|
|
4171
|
+
return [
|
|
4172
|
+
`${error.code}: ${error.message}`,
|
|
4173
|
+
`category=${error.category}`,
|
|
4174
|
+
error.statusCode !== void 0 ? `statusCode=${error.statusCode}` : null,
|
|
4175
|
+
error.details ? `details=${JSON.stringify(error.details)}` : null
|
|
4176
|
+
].filter(Boolean).join(" | ");
|
|
4177
|
+
}
|
|
4178
|
+
|
|
4179
|
+
// src/lib/db.ts
|
|
4111
4180
|
var dbs = {};
|
|
4112
4181
|
function setupDb(sqlitePath) {
|
|
4113
4182
|
const db = new Database(untildify(sqlitePath));
|
|
@@ -4136,22 +4205,39 @@ function openDb(config = null) {
|
|
|
4136
4205
|
return dbs[filename];
|
|
4137
4206
|
}
|
|
4138
4207
|
if (Object.keys(dbs).length > 1) {
|
|
4139
|
-
throw new
|
|
4140
|
-
"Multiple databases open, please specify which one to use."
|
|
4208
|
+
throw new GtfsError(
|
|
4209
|
+
"Multiple databases open, please specify which one to use.",
|
|
4210
|
+
{
|
|
4211
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4212
|
+
category: "database" /* DATABASE */,
|
|
4213
|
+
details: { openDatabaseCount: Object.keys(dbs).length }
|
|
4214
|
+
}
|
|
4141
4215
|
);
|
|
4142
4216
|
}
|
|
4143
|
-
throw new
|
|
4217
|
+
throw new GtfsError("Unable to find database connection.", {
|
|
4218
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4219
|
+
category: "database" /* DATABASE */
|
|
4220
|
+
});
|
|
4144
4221
|
}
|
|
4145
4222
|
function closeDb(db = null) {
|
|
4146
4223
|
if (Object.keys(dbs).length === 0) {
|
|
4147
|
-
throw new
|
|
4148
|
-
"No database connection. Call `openDb(config)` before using any methods."
|
|
4224
|
+
throw new GtfsError(
|
|
4225
|
+
"No database connection. Call `openDb(config)` before using any methods.",
|
|
4226
|
+
{
|
|
4227
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4228
|
+
category: "database" /* DATABASE */
|
|
4229
|
+
}
|
|
4149
4230
|
);
|
|
4150
4231
|
}
|
|
4151
4232
|
if (!db) {
|
|
4152
4233
|
if (Object.keys(dbs).length > 1) {
|
|
4153
|
-
throw new
|
|
4154
|
-
"Multiple database connections. Pass the db you want to close as a parameter to `closeDb`."
|
|
4234
|
+
throw new GtfsError(
|
|
4235
|
+
"Multiple database connections. Pass the db you want to close as a parameter to `closeDb`.",
|
|
4236
|
+
{
|
|
4237
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4238
|
+
category: "database" /* DATABASE */,
|
|
4239
|
+
details: { openDatabaseCount: Object.keys(dbs).length }
|
|
4240
|
+
}
|
|
4155
4241
|
);
|
|
4156
4242
|
}
|
|
4157
4243
|
db = dbs[Object.keys(dbs)[0]];
|
|
@@ -4191,12 +4277,21 @@ import sqlString from "sqlstring-sqlite";
|
|
|
4191
4277
|
import Long from "long";
|
|
4192
4278
|
function validateConfigForImport(config) {
|
|
4193
4279
|
if (!config.agencies || config.agencies.length === 0) {
|
|
4194
|
-
throw new
|
|
4280
|
+
throw new GtfsError("No `agencies` specified in config", {
|
|
4281
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4282
|
+
category: "config" /* CONFIG */,
|
|
4283
|
+
details: { field: "agencies" }
|
|
4284
|
+
});
|
|
4195
4285
|
}
|
|
4196
4286
|
for (const [index, agency2] of config.agencies.entries()) {
|
|
4197
4287
|
if (!agency2.path && !agency2.url) {
|
|
4198
|
-
throw new
|
|
4199
|
-
`No Agency \`url\` or \`path\` specified in config for agency index ${index}
|
|
4288
|
+
throw new GtfsError(
|
|
4289
|
+
`No Agency \`url\` or \`path\` specified in config for agency index ${index}.`,
|
|
4290
|
+
{
|
|
4291
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4292
|
+
category: "config" /* CONFIG */,
|
|
4293
|
+
details: { agencyIndex: index }
|
|
4294
|
+
}
|
|
4200
4295
|
);
|
|
4201
4296
|
}
|
|
4202
4297
|
}
|
|
@@ -4303,7 +4398,16 @@ async function fetchGtfsRealtimeData(type, task) {
|
|
|
4303
4398
|
signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
|
|
4304
4399
|
});
|
|
4305
4400
|
if (response.status !== 200) {
|
|
4306
|
-
throw new
|
|
4401
|
+
throw new GtfsError(`HTTP ${response.status}: ${response.statusText}`, {
|
|
4402
|
+
code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
|
|
4403
|
+
category: "download" /* DOWNLOAD */,
|
|
4404
|
+
statusCode: response.status,
|
|
4405
|
+
details: {
|
|
4406
|
+
url: urlConfig.url,
|
|
4407
|
+
status: response.status,
|
|
4408
|
+
statusText: response.statusText
|
|
4409
|
+
}
|
|
4410
|
+
});
|
|
4307
4411
|
}
|
|
4308
4412
|
const buffer = await response.arrayBuffer();
|
|
4309
4413
|
const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
|
|
@@ -4320,17 +4424,27 @@ async function fetchGtfsRealtimeData(type, task) {
|
|
|
4320
4424
|
});
|
|
4321
4425
|
return feedMessage;
|
|
4322
4426
|
} catch (error) {
|
|
4323
|
-
const
|
|
4427
|
+
const gtfsError = toGtfsError(error, {
|
|
4428
|
+
message: error instanceof Error ? error.message : String(error),
|
|
4429
|
+
code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
|
|
4430
|
+
category: "download" /* DOWNLOAD */,
|
|
4431
|
+
details: { type, url: urlConfig.url }
|
|
4432
|
+
});
|
|
4324
4433
|
if (attempt === MAX_RETRIES) {
|
|
4325
4434
|
if (task.ignoreErrors) {
|
|
4326
4435
|
task.logError(
|
|
4327
|
-
`Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${
|
|
4436
|
+
`Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${gtfsError.message}`
|
|
4328
4437
|
);
|
|
4438
|
+
if (task.report) {
|
|
4439
|
+
addImportError(task.report, gtfsError);
|
|
4440
|
+
}
|
|
4329
4441
|
return null;
|
|
4330
4442
|
}
|
|
4331
|
-
throw
|
|
4443
|
+
throw gtfsError;
|
|
4332
4444
|
}
|
|
4333
|
-
task.logWarning(
|
|
4445
|
+
task.logWarning(
|
|
4446
|
+
`Attempt ${attempt} failed for ${type}: ${gtfsError.message}`
|
|
4447
|
+
);
|
|
4334
4448
|
await new Promise(
|
|
4335
4449
|
(resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
|
|
4336
4450
|
);
|
|
@@ -4505,33 +4619,64 @@ async function updateGtfsRealtimeData(task) {
|
|
|
4505
4619
|
}
|
|
4506
4620
|
|
|
4507
4621
|
// src/lib/import-gtfs.ts
|
|
4622
|
+
function reportTaskError(task, error) {
|
|
4623
|
+
if (task.report) {
|
|
4624
|
+
addImportError(task.report, error);
|
|
4625
|
+
}
|
|
4626
|
+
}
|
|
4508
4627
|
var getTextFiles = async (folderPath) => {
|
|
4509
4628
|
const files = await readdir(folderPath);
|
|
4510
4629
|
return files.filter((filename) => filename.slice(-3) === "txt");
|
|
4511
4630
|
};
|
|
4512
4631
|
var downloadGtfsFiles = async (task) => {
|
|
4513
4632
|
if (!task.url) {
|
|
4514
|
-
throw new
|
|
4633
|
+
throw new GtfsError("No `url` specified in config", {
|
|
4634
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4635
|
+
category: "config" /* CONFIG */
|
|
4636
|
+
});
|
|
4515
4637
|
}
|
|
4516
4638
|
task.log(`Downloading GTFS from ${task.url}`);
|
|
4517
4639
|
task.path = `${task.downloadDir}/gtfs.zip`;
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4640
|
+
try {
|
|
4641
|
+
const response = await fetch(task.url, {
|
|
4642
|
+
method: "GET",
|
|
4643
|
+
headers: task.headers || {},
|
|
4644
|
+
signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
|
|
4645
|
+
});
|
|
4646
|
+
if (response.status !== 200) {
|
|
4647
|
+
throw new GtfsError(
|
|
4648
|
+
`Unable to download GTFS from ${task.url}. Got status ${response.status}.`,
|
|
4649
|
+
{
|
|
4650
|
+
code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
|
|
4651
|
+
category: "download" /* DOWNLOAD */,
|
|
4652
|
+
statusCode: response.status,
|
|
4653
|
+
details: {
|
|
4654
|
+
url: task.url,
|
|
4655
|
+
status: response.status,
|
|
4656
|
+
statusText: response.statusText
|
|
4657
|
+
}
|
|
4658
|
+
}
|
|
4659
|
+
);
|
|
4660
|
+
}
|
|
4661
|
+
const buffer = await response.arrayBuffer();
|
|
4662
|
+
await writeFile(task.path, Buffer.from(buffer));
|
|
4663
|
+
task.log("Download successful");
|
|
4664
|
+
} catch (error) {
|
|
4665
|
+
throw toGtfsError(error, {
|
|
4666
|
+
message: `Unable to download GTFS from ${task.url}.`,
|
|
4667
|
+
code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
|
|
4668
|
+
category: "download" /* DOWNLOAD */,
|
|
4669
|
+
details: { url: task.url }
|
|
4670
|
+
});
|
|
4527
4671
|
}
|
|
4528
|
-
const buffer = await response.arrayBuffer();
|
|
4529
|
-
await writeFile(task.path, Buffer.from(buffer));
|
|
4530
|
-
task.log("Download successful");
|
|
4531
4672
|
};
|
|
4532
4673
|
var extractGtfsFiles = async (task) => {
|
|
4533
4674
|
if (!task.path) {
|
|
4534
|
-
throw new
|
|
4675
|
+
throw new GtfsError("No `path` specified in config", {
|
|
4676
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4677
|
+
category: "config" /* CONFIG */,
|
|
4678
|
+
details: { field: "path" }
|
|
4679
|
+
});
|
|
4535
4680
|
}
|
|
4536
4681
|
const gtfsPath = untildify(task.path);
|
|
4537
4682
|
task.log(`Importing static GTFS from ${task.path}\r`);
|
|
@@ -4543,19 +4688,34 @@ var extractGtfsFiles = async (task) => {
|
|
|
4543
4688
|
const files = await readdir(task.downloadDir);
|
|
4544
4689
|
const folders = files.filter((filename) => !["__MACOSX"].includes(filename)).map((filename) => path2.join(task.downloadDir, filename)).filter((source) => lstatSync(source).isDirectory());
|
|
4545
4690
|
if (folders.length > 1) {
|
|
4546
|
-
throw new
|
|
4547
|
-
`More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory
|
|
4691
|
+
throw new GtfsError(
|
|
4692
|
+
`More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
|
|
4693
|
+
{
|
|
4694
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4695
|
+
category: "zip" /* ZIP */,
|
|
4696
|
+
details: { path: task.path, folderCount: folders.length }
|
|
4697
|
+
}
|
|
4548
4698
|
);
|
|
4549
4699
|
} else if (folders.length === 0) {
|
|
4550
|
-
throw new
|
|
4551
|
-
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory
|
|
4700
|
+
throw new GtfsError(
|
|
4701
|
+
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
|
|
4702
|
+
{
|
|
4703
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4704
|
+
category: "zip" /* ZIP */,
|
|
4705
|
+
details: { path: task.path }
|
|
4706
|
+
}
|
|
4552
4707
|
);
|
|
4553
4708
|
}
|
|
4554
4709
|
const subfolderName = folders[0];
|
|
4555
4710
|
const directoryTextFiles = await getTextFiles(subfolderName);
|
|
4556
4711
|
if (directoryTextFiles.length === 0) {
|
|
4557
|
-
throw new
|
|
4558
|
-
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory
|
|
4712
|
+
throw new GtfsError(
|
|
4713
|
+
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
|
|
4714
|
+
{
|
|
4715
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4716
|
+
category: "zip" /* ZIP */,
|
|
4717
|
+
details: { path: task.path, subfolderName }
|
|
4718
|
+
}
|
|
4559
4719
|
);
|
|
4560
4720
|
}
|
|
4561
4721
|
await Promise.all(
|
|
@@ -4568,15 +4728,27 @@ var extractGtfsFiles = async (task) => {
|
|
|
4568
4728
|
);
|
|
4569
4729
|
}
|
|
4570
4730
|
} catch (error) {
|
|
4571
|
-
|
|
4572
|
-
|
|
4731
|
+
const wrappedError = toGtfsError(error, {
|
|
4732
|
+
message: `Unable to unzip file ${task.path}`,
|
|
4733
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4734
|
+
category: "zip" /* ZIP */,
|
|
4735
|
+
details: { path: task.path }
|
|
4736
|
+
});
|
|
4737
|
+
task.logError(formatGtfsError(wrappedError));
|
|
4738
|
+
throw wrappedError;
|
|
4573
4739
|
}
|
|
4574
4740
|
} else {
|
|
4575
4741
|
try {
|
|
4576
4742
|
await cp(gtfsPath, task.downloadDir, { recursive: true });
|
|
4577
|
-
} catch {
|
|
4578
|
-
throw new
|
|
4579
|
-
`Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files
|
|
4743
|
+
} catch (error) {
|
|
4744
|
+
throw new GtfsError(
|
|
4745
|
+
`Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`,
|
|
4746
|
+
{
|
|
4747
|
+
code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
|
|
4748
|
+
category: "download" /* DOWNLOAD */,
|
|
4749
|
+
details: { path: gtfsPath },
|
|
4750
|
+
cause: error
|
|
4751
|
+
}
|
|
4580
4752
|
);
|
|
4581
4753
|
}
|
|
4582
4754
|
}
|
|
@@ -4671,8 +4843,17 @@ var formatGtfsLine = (line, model, totalLineCount) => {
|
|
|
4671
4843
|
if (value === "" || value === void 0 || value === null) {
|
|
4672
4844
|
formattedLine[name] = null;
|
|
4673
4845
|
if (required) {
|
|
4674
|
-
throw new
|
|
4675
|
-
`Missing required value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}
|
|
4846
|
+
throw new GtfsError(
|
|
4847
|
+
`Missing required value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`,
|
|
4848
|
+
{
|
|
4849
|
+
code: "GTFS_REQUIRED_FIELD_MISSING" /* GTFS_REQUIRED_FIELD_MISSING */,
|
|
4850
|
+
category: "validation" /* VALIDATION */,
|
|
4851
|
+
details: {
|
|
4852
|
+
file: `${filenameBase}.${filenameExtension}`,
|
|
4853
|
+
line: lineNumber,
|
|
4854
|
+
column: name
|
|
4855
|
+
}
|
|
4856
|
+
}
|
|
4676
4857
|
);
|
|
4677
4858
|
}
|
|
4678
4859
|
continue;
|
|
@@ -4680,8 +4861,18 @@ var formatGtfsLine = (line, model, totalLineCount) => {
|
|
|
4680
4861
|
if (type === "date") {
|
|
4681
4862
|
value = value?.toString().replace(/-/g, "");
|
|
4682
4863
|
if (value.length !== 8) {
|
|
4683
|
-
throw new
|
|
4684
|
-
`Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}
|
|
4864
|
+
throw new GtfsError(
|
|
4865
|
+
`Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`,
|
|
4866
|
+
{
|
|
4867
|
+
code: "GTFS_INVALID_DATE" /* GTFS_INVALID_DATE */,
|
|
4868
|
+
category: "validation" /* VALIDATION */,
|
|
4869
|
+
details: {
|
|
4870
|
+
file: `${filenameBase}.${filenameExtension}`,
|
|
4871
|
+
line: lineNumber,
|
|
4872
|
+
column: name,
|
|
4873
|
+
value
|
|
4874
|
+
}
|
|
4875
|
+
}
|
|
4685
4876
|
);
|
|
4686
4877
|
}
|
|
4687
4878
|
} else if (type === "time") {
|
|
@@ -4753,11 +4944,32 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4753
4944
|
task.logWarning(
|
|
4754
4945
|
`Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
|
|
4755
4946
|
);
|
|
4947
|
+
if (task.report) {
|
|
4948
|
+
addImportWarning(task.report, {
|
|
4949
|
+
code: "GTFS_DUPLICATE_PRIMARY_KEY" /* GTFS_DUPLICATE_PRIMARY_KEY */,
|
|
4950
|
+
message: `Duplicate values for primary key found in ${filename}.`,
|
|
4951
|
+
details: {
|
|
4952
|
+
file: filename,
|
|
4953
|
+
line: Number(rowNumber) + 1,
|
|
4954
|
+
columns: primaryColumns.map((column) => column.name)
|
|
4955
|
+
}
|
|
4956
|
+
});
|
|
4957
|
+
}
|
|
4756
4958
|
}
|
|
4757
4959
|
task.logWarning(
|
|
4758
4960
|
`Check ${filename} for invalid data on line ${rowNumber + 1}.`
|
|
4759
4961
|
);
|
|
4760
|
-
throw error
|
|
4962
|
+
throw toGtfsError(error, {
|
|
4963
|
+
message: error instanceof Error ? error.message : String(error),
|
|
4964
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4965
|
+
category: "database" /* DATABASE */,
|
|
4966
|
+
details: {
|
|
4967
|
+
file: filename,
|
|
4968
|
+
line: Number(rowNumber) + 1,
|
|
4969
|
+
sqlitePath: task.sqlitePath,
|
|
4970
|
+
dbCode: error.code
|
|
4971
|
+
}
|
|
4972
|
+
});
|
|
4761
4973
|
}
|
|
4762
4974
|
}
|
|
4763
4975
|
});
|
|
@@ -4786,12 +4998,20 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4786
4998
|
}
|
|
4787
4999
|
}
|
|
4788
5000
|
} catch (error) {
|
|
5001
|
+
const gtfsError = toGtfsError(error, {
|
|
5002
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5003
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5004
|
+
category: "parse" /* PARSE */,
|
|
5005
|
+
details: { file: filename }
|
|
5006
|
+
});
|
|
4789
5007
|
if (task.ignoreErrors) {
|
|
4790
|
-
|
|
4791
|
-
task.logError(
|
|
5008
|
+
reportTaskError(task, gtfsError);
|
|
5009
|
+
task.logError(
|
|
5010
|
+
`Error processing ${filename}: ${gtfsError.message}`
|
|
5011
|
+
);
|
|
4792
5012
|
resolve();
|
|
4793
5013
|
} else {
|
|
4794
|
-
reject(
|
|
5014
|
+
reject(gtfsError);
|
|
4795
5015
|
}
|
|
4796
5016
|
}
|
|
4797
5017
|
});
|
|
@@ -4801,15 +5021,21 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4801
5021
|
try {
|
|
4802
5022
|
insertLines(lines);
|
|
4803
5023
|
} catch (error) {
|
|
5024
|
+
const gtfsError = toGtfsError(error, {
|
|
5025
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5026
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
5027
|
+
category: "database" /* DATABASE */,
|
|
5028
|
+
details: { file: filename, sqlitePath: task.sqlitePath }
|
|
5029
|
+
});
|
|
4804
5030
|
if (task.ignoreErrors) {
|
|
4805
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
4806
5031
|
task.logError(
|
|
4807
|
-
`Error inserting data for ${filename}: ${
|
|
5032
|
+
`Error inserting data for ${filename}: ${gtfsError.message}`
|
|
4808
5033
|
);
|
|
5034
|
+
reportTaskError(task, gtfsError);
|
|
4809
5035
|
resolve();
|
|
4810
5036
|
return;
|
|
4811
5037
|
} else {
|
|
4812
|
-
reject(
|
|
5038
|
+
reject(gtfsError);
|
|
4813
5039
|
return;
|
|
4814
5040
|
}
|
|
4815
5041
|
}
|
|
@@ -4820,22 +5046,38 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4820
5046
|
);
|
|
4821
5047
|
resolve();
|
|
4822
5048
|
} catch (error) {
|
|
5049
|
+
const gtfsError = toGtfsError(error, {
|
|
5050
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5051
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
5052
|
+
category: "database" /* DATABASE */,
|
|
5053
|
+
details: { file: filename, sqlitePath: task.sqlitePath }
|
|
5054
|
+
});
|
|
4823
5055
|
if (task.ignoreErrors) {
|
|
4824
|
-
|
|
4825
|
-
|
|
5056
|
+
task.logError(
|
|
5057
|
+
`Error finalizing ${filename}: ${gtfsError.message}`
|
|
5058
|
+
);
|
|
5059
|
+
reportTaskError(task, gtfsError);
|
|
4826
5060
|
resolve();
|
|
4827
5061
|
} else {
|
|
4828
|
-
reject(
|
|
5062
|
+
reject(gtfsError);
|
|
4829
5063
|
}
|
|
4830
5064
|
}
|
|
4831
5065
|
});
|
|
4832
5066
|
parser.on("error", (error) => {
|
|
5067
|
+
const gtfsError = toGtfsError(error, {
|
|
5068
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5069
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5070
|
+
category: "parse" /* PARSE */,
|
|
5071
|
+
details: { file: filename }
|
|
5072
|
+
});
|
|
4833
5073
|
if (task.ignoreErrors) {
|
|
4834
|
-
|
|
4835
|
-
|
|
5074
|
+
task.logError(
|
|
5075
|
+
`Parser error for ${filename}: ${gtfsError.message}`
|
|
5076
|
+
);
|
|
5077
|
+
reportTaskError(task, gtfsError);
|
|
4836
5078
|
resolve();
|
|
4837
5079
|
} else {
|
|
4838
|
-
reject(
|
|
5080
|
+
reject(gtfsError);
|
|
4839
5081
|
}
|
|
4840
5082
|
});
|
|
4841
5083
|
createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
|
|
@@ -4844,10 +5086,24 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4844
5086
|
if (isValidJSON(data) === false) {
|
|
4845
5087
|
if (task.ignoreErrors) {
|
|
4846
5088
|
task.logError(`Invalid JSON in ${filename}`);
|
|
5089
|
+
reportTaskError(
|
|
5090
|
+
task,
|
|
5091
|
+
new GtfsError(`Invalid JSON in ${filename}`, {
|
|
5092
|
+
code: "GTFS_JSON_INVALID" /* GTFS_JSON_INVALID */,
|
|
5093
|
+
category: "parse" /* PARSE */,
|
|
5094
|
+
details: { file: filename }
|
|
5095
|
+
})
|
|
5096
|
+
);
|
|
4847
5097
|
resolve();
|
|
4848
5098
|
return;
|
|
4849
5099
|
} else {
|
|
4850
|
-
reject(
|
|
5100
|
+
reject(
|
|
5101
|
+
new GtfsError(`Invalid JSON in ${filename}`, {
|
|
5102
|
+
code: "GTFS_JSON_INVALID" /* GTFS_JSON_INVALID */,
|
|
5103
|
+
category: "parse" /* PARSE */,
|
|
5104
|
+
details: { file: filename }
|
|
5105
|
+
})
|
|
5106
|
+
);
|
|
4851
5107
|
return;
|
|
4852
5108
|
}
|
|
4853
5109
|
}
|
|
@@ -4865,23 +5121,37 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4865
5121
|
);
|
|
4866
5122
|
resolve();
|
|
4867
5123
|
} catch (error) {
|
|
5124
|
+
const gtfsError = toGtfsError(error, {
|
|
5125
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5126
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
5127
|
+
category: "database" /* DATABASE */,
|
|
5128
|
+
details: { file: filename, sqlitePath: task.sqlitePath }
|
|
5129
|
+
});
|
|
4868
5130
|
if (task.ignoreErrors) {
|
|
4869
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
4870
5131
|
task.logError(
|
|
4871
|
-
`Error inserting data for ${filename}: ${
|
|
5132
|
+
`Error inserting data for ${filename}: ${gtfsError.message}`
|
|
4872
5133
|
);
|
|
5134
|
+
reportTaskError(task, gtfsError);
|
|
4873
5135
|
resolve();
|
|
4874
5136
|
} else {
|
|
4875
|
-
reject(
|
|
5137
|
+
reject(gtfsError);
|
|
4876
5138
|
}
|
|
4877
5139
|
}
|
|
4878
5140
|
}).catch((error) => {
|
|
5141
|
+
const gtfsError = toGtfsError(error, {
|
|
5142
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5143
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5144
|
+
category: "parse" /* PARSE */,
|
|
5145
|
+
details: { file: filename }
|
|
5146
|
+
});
|
|
4879
5147
|
if (task.ignoreErrors) {
|
|
4880
|
-
|
|
4881
|
-
|
|
5148
|
+
task.logError(
|
|
5149
|
+
`Error reading ${filename}: ${gtfsError.message}`
|
|
5150
|
+
);
|
|
5151
|
+
reportTaskError(task, gtfsError);
|
|
4882
5152
|
resolve();
|
|
4883
5153
|
} else {
|
|
4884
|
-
reject(
|
|
5154
|
+
reject(gtfsError);
|
|
4885
5155
|
}
|
|
4886
5156
|
});
|
|
4887
5157
|
} else {
|
|
@@ -4892,7 +5162,17 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4892
5162
|
resolve();
|
|
4893
5163
|
} else {
|
|
4894
5164
|
reject(
|
|
4895
|
-
new
|
|
5165
|
+
new GtfsError(
|
|
5166
|
+
`Unsupported file type: ${model.filenameExtension}`,
|
|
5167
|
+
{
|
|
5168
|
+
code: "GTFS_UNSUPPORTED_FILE_TYPE" /* GTFS_UNSUPPORTED_FILE_TYPE */,
|
|
5169
|
+
category: "parse" /* PARSE */,
|
|
5170
|
+
details: {
|
|
5171
|
+
file: filename,
|
|
5172
|
+
extension: model.filenameExtension
|
|
5173
|
+
}
|
|
5174
|
+
}
|
|
5175
|
+
)
|
|
4896
5176
|
);
|
|
4897
5177
|
}
|
|
4898
5178
|
}
|
|
@@ -4904,6 +5184,7 @@ async function importGtfs(initialConfig) {
|
|
|
4904
5184
|
const startTime = process.hrtime.bigint();
|
|
4905
5185
|
const config = setDefaultConfig(initialConfig);
|
|
4906
5186
|
validateConfigForImport(config);
|
|
5187
|
+
const report = config.includeImportReport ? createImportReport() : void 0;
|
|
4907
5188
|
try {
|
|
4908
5189
|
const db = openDb(config);
|
|
4909
5190
|
const agencyCount = config.agencies.length;
|
|
@@ -4931,7 +5212,8 @@ async function importGtfs(initialConfig) {
|
|
|
4931
5212
|
currentTimestamp: Math.floor(Date.now() / 1e3),
|
|
4932
5213
|
log: log(config),
|
|
4933
5214
|
logWarning: logWarning(config),
|
|
4934
|
-
logError: logError(config)
|
|
5215
|
+
logError: logError(config),
|
|
5216
|
+
report
|
|
4935
5217
|
};
|
|
4936
5218
|
if ("url" in agency2) {
|
|
4937
5219
|
Object.assign(task, { url: agency2.url });
|
|
@@ -4946,11 +5228,18 @@ async function importGtfs(initialConfig) {
|
|
|
4946
5228
|
await updateGtfsRealtimeData(task);
|
|
4947
5229
|
await rm2(tempPath, { recursive: true });
|
|
4948
5230
|
} catch (error) {
|
|
5231
|
+
const wrappedError = toGtfsError(error, {
|
|
5232
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5233
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5234
|
+
category: "parse" /* PARSE */
|
|
5235
|
+
});
|
|
4949
5236
|
if (config.ignoreErrors) {
|
|
4950
|
-
|
|
4951
|
-
|
|
5237
|
+
logError(config)(formatGtfsError(wrappedError));
|
|
5238
|
+
if (report) {
|
|
5239
|
+
addImportError(report, wrappedError);
|
|
5240
|
+
}
|
|
4952
5241
|
} else {
|
|
4953
|
-
throw
|
|
5242
|
+
throw wrappedError;
|
|
4954
5243
|
}
|
|
4955
5244
|
}
|
|
4956
5245
|
});
|
|
@@ -4964,11 +5253,29 @@ async function importGtfs(initialConfig) {
|
|
|
4964
5253
|
);
|
|
4965
5254
|
} catch (error) {
|
|
4966
5255
|
if (error.code === "SQLITE_CANTOPEN") {
|
|
4967
|
-
|
|
4968
|
-
`Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json
|
|
5256
|
+
const dbOpenError = new GtfsError(
|
|
5257
|
+
`Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`,
|
|
5258
|
+
{
|
|
5259
|
+
code: "DB_OPEN_FAILED" /* DB_OPEN_FAILED */,
|
|
5260
|
+
category: "database" /* DATABASE */,
|
|
5261
|
+
details: {
|
|
5262
|
+
sqlitePath: config.sqlitePath,
|
|
5263
|
+
dbCode: error.code
|
|
5264
|
+
},
|
|
5265
|
+
cause: error
|
|
5266
|
+
}
|
|
4969
5267
|
);
|
|
5268
|
+
logError(config)(dbOpenError.message);
|
|
5269
|
+
throw dbOpenError;
|
|
4970
5270
|
}
|
|
4971
|
-
throw error
|
|
5271
|
+
throw toGtfsError(error, {
|
|
5272
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5273
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5274
|
+
category: "parse" /* PARSE */
|
|
5275
|
+
});
|
|
5276
|
+
}
|
|
5277
|
+
if (report) {
|
|
5278
|
+
return report;
|
|
4972
5279
|
}
|
|
4973
5280
|
}
|
|
4974
5281
|
|