gtfs 4.18.2 → 4.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/gtfs-export.js +31 -3
- package/dist/bin/gtfs-export.js.map +1 -1
- package/dist/bin/gtfs-import.js +377 -72
- package/dist/bin/gtfs-import.js.map +1 -1
- package/dist/bin/gtfsrealtime-update.js +133 -18
- package/dist/bin/gtfsrealtime-update.js.map +1 -1
- package/dist/index.d.ts +73 -1
- package/dist/index.js +499 -91
- package/dist/index.js.map +1 -1
- package/package.json +14 -14
package/dist/bin/gtfs-import.js
CHANGED
|
@@ -4108,6 +4108,73 @@ var vehicles = {
|
|
|
4108
4108
|
|
|
4109
4109
|
// src/lib/db.ts
|
|
4110
4110
|
import Database from "better-sqlite3";
|
|
4111
|
+
|
|
4112
|
+
// src/lib/errors.ts
|
|
4113
|
+
var GtfsError = class extends Error {
|
|
4114
|
+
code;
|
|
4115
|
+
category;
|
|
4116
|
+
isOperational;
|
|
4117
|
+
statusCode;
|
|
4118
|
+
details;
|
|
4119
|
+
constructor(message, options) {
|
|
4120
|
+
super(message, { cause: options.cause });
|
|
4121
|
+
this.name = "GtfsError";
|
|
4122
|
+
this.code = options.code;
|
|
4123
|
+
this.category = options.category;
|
|
4124
|
+
this.isOperational = options.isOperational ?? true;
|
|
4125
|
+
this.statusCode = options.statusCode;
|
|
4126
|
+
this.details = options.details;
|
|
4127
|
+
}
|
|
4128
|
+
};
|
|
4129
|
+
function isGtfsError(error) {
|
|
4130
|
+
if (!error || typeof error !== "object") {
|
|
4131
|
+
return false;
|
|
4132
|
+
}
|
|
4133
|
+
const candidate = error;
|
|
4134
|
+
return candidate.name === "GtfsError" && typeof candidate.message === "string" && typeof candidate.code === "string" && typeof candidate.category === "string" && typeof candidate.isOperational === "boolean";
|
|
4135
|
+
}
|
|
4136
|
+
function toGtfsError(error, fallback) {
|
|
4137
|
+
if (isGtfsError(error)) {
|
|
4138
|
+
return error;
|
|
4139
|
+
}
|
|
4140
|
+
return new GtfsError(fallback.message, {
|
|
4141
|
+
...fallback,
|
|
4142
|
+
cause: error
|
|
4143
|
+
});
|
|
4144
|
+
}
|
|
4145
|
+
function createImportReport() {
|
|
4146
|
+
return {
|
|
4147
|
+
errors: [],
|
|
4148
|
+
warnings: [],
|
|
4149
|
+
errorCountsByCode: {},
|
|
4150
|
+
warningCountsByCode: {}
|
|
4151
|
+
};
|
|
4152
|
+
}
|
|
4153
|
+
function addImportError(report, error) {
|
|
4154
|
+
report.errors.push(error);
|
|
4155
|
+
report.errorCountsByCode[error.code] = (report.errorCountsByCode[error.code] ?? 0) + 1;
|
|
4156
|
+
}
|
|
4157
|
+
function addImportWarning(report, warning) {
|
|
4158
|
+
report.warnings.push(warning);
|
|
4159
|
+
report.warningCountsByCode[warning.code] = (report.warningCountsByCode[warning.code] ?? 0) + 1;
|
|
4160
|
+
}
|
|
4161
|
+
function formatGtfsError(error, options = { verbosity: "developer" }) {
|
|
4162
|
+
if (!isGtfsError(error)) {
|
|
4163
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4164
|
+
return options.verbosity === "user" ? message : `UNKNOWN_ERROR: ${message}`;
|
|
4165
|
+
}
|
|
4166
|
+
if (options.verbosity === "user") {
|
|
4167
|
+
return error.message;
|
|
4168
|
+
}
|
|
4169
|
+
return [
|
|
4170
|
+
`${error.code}: ${error.message}`,
|
|
4171
|
+
`category=${error.category}`,
|
|
4172
|
+
error.statusCode !== void 0 ? `statusCode=${error.statusCode}` : null,
|
|
4173
|
+
error.details ? `details=${JSON.stringify(error.details)}` : null
|
|
4174
|
+
].filter(Boolean).join(" | ");
|
|
4175
|
+
}
|
|
4176
|
+
|
|
4177
|
+
// src/lib/db.ts
|
|
4111
4178
|
var dbs = {};
|
|
4112
4179
|
function setupDb(sqlitePath) {
|
|
4113
4180
|
const db = new Database(untildify(sqlitePath));
|
|
@@ -4136,22 +4203,39 @@ function openDb(config = null) {
|
|
|
4136
4203
|
return dbs[filename];
|
|
4137
4204
|
}
|
|
4138
4205
|
if (Object.keys(dbs).length > 1) {
|
|
4139
|
-
throw new
|
|
4140
|
-
"Multiple databases open, please specify which one to use."
|
|
4206
|
+
throw new GtfsError(
|
|
4207
|
+
"Multiple databases open, please specify which one to use.",
|
|
4208
|
+
{
|
|
4209
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4210
|
+
category: "database" /* DATABASE */,
|
|
4211
|
+
details: { openDatabaseCount: Object.keys(dbs).length }
|
|
4212
|
+
}
|
|
4141
4213
|
);
|
|
4142
4214
|
}
|
|
4143
|
-
throw new
|
|
4215
|
+
throw new GtfsError("Unable to find database connection.", {
|
|
4216
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4217
|
+
category: "database" /* DATABASE */
|
|
4218
|
+
});
|
|
4144
4219
|
}
|
|
4145
4220
|
function closeDb(db = null) {
|
|
4146
4221
|
if (Object.keys(dbs).length === 0) {
|
|
4147
|
-
throw new
|
|
4148
|
-
"No database connection. Call `openDb(config)` before using any methods."
|
|
4222
|
+
throw new GtfsError(
|
|
4223
|
+
"No database connection. Call `openDb(config)` before using any methods.",
|
|
4224
|
+
{
|
|
4225
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4226
|
+
category: "database" /* DATABASE */
|
|
4227
|
+
}
|
|
4149
4228
|
);
|
|
4150
4229
|
}
|
|
4151
4230
|
if (!db) {
|
|
4152
4231
|
if (Object.keys(dbs).length > 1) {
|
|
4153
|
-
throw new
|
|
4154
|
-
"Multiple database connections. Pass the db you want to close as a parameter to `closeDb`."
|
|
4232
|
+
throw new GtfsError(
|
|
4233
|
+
"Multiple database connections. Pass the db you want to close as a parameter to `closeDb`.",
|
|
4234
|
+
{
|
|
4235
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4236
|
+
category: "database" /* DATABASE */,
|
|
4237
|
+
details: { openDatabaseCount: Object.keys(dbs).length }
|
|
4238
|
+
}
|
|
4155
4239
|
);
|
|
4156
4240
|
}
|
|
4157
4241
|
db = dbs[Object.keys(dbs)[0]];
|
|
@@ -4191,12 +4275,21 @@ import sqlString from "sqlstring-sqlite";
|
|
|
4191
4275
|
import Long from "long";
|
|
4192
4276
|
function validateConfigForImport(config) {
|
|
4193
4277
|
if (!config.agencies || config.agencies.length === 0) {
|
|
4194
|
-
throw new
|
|
4278
|
+
throw new GtfsError("No `agencies` specified in config", {
|
|
4279
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4280
|
+
category: "config" /* CONFIG */,
|
|
4281
|
+
details: { field: "agencies" }
|
|
4282
|
+
});
|
|
4195
4283
|
}
|
|
4196
4284
|
for (const [index, agency2] of config.agencies.entries()) {
|
|
4197
4285
|
if (!agency2.path && !agency2.url) {
|
|
4198
|
-
throw new
|
|
4199
|
-
`No Agency \`url\` or \`path\` specified in config for agency index ${index}
|
|
4286
|
+
throw new GtfsError(
|
|
4287
|
+
`No Agency \`url\` or \`path\` specified in config for agency index ${index}.`,
|
|
4288
|
+
{
|
|
4289
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4290
|
+
category: "config" /* CONFIG */,
|
|
4291
|
+
details: { agencyIndex: index }
|
|
4292
|
+
}
|
|
4200
4293
|
);
|
|
4201
4294
|
}
|
|
4202
4295
|
}
|
|
@@ -4303,7 +4396,16 @@ async function fetchGtfsRealtimeData(type, task) {
|
|
|
4303
4396
|
signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
|
|
4304
4397
|
});
|
|
4305
4398
|
if (response.status !== 200) {
|
|
4306
|
-
throw new
|
|
4399
|
+
throw new GtfsError(`HTTP ${response.status}: ${response.statusText}`, {
|
|
4400
|
+
code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
|
|
4401
|
+
category: "download" /* DOWNLOAD */,
|
|
4402
|
+
statusCode: response.status,
|
|
4403
|
+
details: {
|
|
4404
|
+
url: urlConfig.url,
|
|
4405
|
+
status: response.status,
|
|
4406
|
+
statusText: response.statusText
|
|
4407
|
+
}
|
|
4408
|
+
});
|
|
4307
4409
|
}
|
|
4308
4410
|
const buffer = await response.arrayBuffer();
|
|
4309
4411
|
const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
|
|
@@ -4320,17 +4422,27 @@ async function fetchGtfsRealtimeData(type, task) {
|
|
|
4320
4422
|
});
|
|
4321
4423
|
return feedMessage;
|
|
4322
4424
|
} catch (error) {
|
|
4323
|
-
const
|
|
4425
|
+
const gtfsError = toGtfsError(error, {
|
|
4426
|
+
message: error instanceof Error ? error.message : String(error),
|
|
4427
|
+
code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
|
|
4428
|
+
category: "download" /* DOWNLOAD */,
|
|
4429
|
+
details: { type, url: urlConfig.url }
|
|
4430
|
+
});
|
|
4324
4431
|
if (attempt === MAX_RETRIES) {
|
|
4325
4432
|
if (task.ignoreErrors) {
|
|
4326
4433
|
task.logError(
|
|
4327
|
-
`Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${
|
|
4434
|
+
`Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${gtfsError.message}`
|
|
4328
4435
|
);
|
|
4436
|
+
if (task.report) {
|
|
4437
|
+
addImportError(task.report, gtfsError);
|
|
4438
|
+
}
|
|
4329
4439
|
return null;
|
|
4330
4440
|
}
|
|
4331
|
-
throw
|
|
4441
|
+
throw gtfsError;
|
|
4332
4442
|
}
|
|
4333
|
-
task.logWarning(
|
|
4443
|
+
task.logWarning(
|
|
4444
|
+
`Attempt ${attempt} failed for ${type}: ${gtfsError.message}`
|
|
4445
|
+
);
|
|
4334
4446
|
await new Promise(
|
|
4335
4447
|
(resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
|
|
4336
4448
|
);
|
|
@@ -4505,33 +4617,64 @@ async function updateGtfsRealtimeData(task) {
|
|
|
4505
4617
|
}
|
|
4506
4618
|
|
|
4507
4619
|
// src/lib/import-gtfs.ts
|
|
4620
|
+
function reportTaskError(task, error) {
|
|
4621
|
+
if (task.report) {
|
|
4622
|
+
addImportError(task.report, error);
|
|
4623
|
+
}
|
|
4624
|
+
}
|
|
4508
4625
|
var getTextFiles = async (folderPath) => {
|
|
4509
4626
|
const files = await readdir(folderPath);
|
|
4510
4627
|
return files.filter((filename) => filename.slice(-3) === "txt");
|
|
4511
4628
|
};
|
|
4512
4629
|
var downloadGtfsFiles = async (task) => {
|
|
4513
4630
|
if (!task.url) {
|
|
4514
|
-
throw new
|
|
4631
|
+
throw new GtfsError("No `url` specified in config", {
|
|
4632
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4633
|
+
category: "config" /* CONFIG */
|
|
4634
|
+
});
|
|
4515
4635
|
}
|
|
4516
4636
|
task.log(`Downloading GTFS from ${task.url}`);
|
|
4517
4637
|
task.path = `${task.downloadDir}/gtfs.zip`;
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4638
|
+
try {
|
|
4639
|
+
const response = await fetch(task.url, {
|
|
4640
|
+
method: "GET",
|
|
4641
|
+
headers: task.headers || {},
|
|
4642
|
+
signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
|
|
4643
|
+
});
|
|
4644
|
+
if (response.status !== 200) {
|
|
4645
|
+
throw new GtfsError(
|
|
4646
|
+
`Unable to download GTFS from ${task.url}. Got status ${response.status}.`,
|
|
4647
|
+
{
|
|
4648
|
+
code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
|
|
4649
|
+
category: "download" /* DOWNLOAD */,
|
|
4650
|
+
statusCode: response.status,
|
|
4651
|
+
details: {
|
|
4652
|
+
url: task.url,
|
|
4653
|
+
status: response.status,
|
|
4654
|
+
statusText: response.statusText
|
|
4655
|
+
}
|
|
4656
|
+
}
|
|
4657
|
+
);
|
|
4658
|
+
}
|
|
4659
|
+
const buffer = await response.arrayBuffer();
|
|
4660
|
+
await writeFile(task.path, Buffer.from(buffer));
|
|
4661
|
+
task.log("Download successful");
|
|
4662
|
+
} catch (error) {
|
|
4663
|
+
throw toGtfsError(error, {
|
|
4664
|
+
message: `Unable to download GTFS from ${task.url}.`,
|
|
4665
|
+
code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
|
|
4666
|
+
category: "download" /* DOWNLOAD */,
|
|
4667
|
+
details: { url: task.url }
|
|
4668
|
+
});
|
|
4527
4669
|
}
|
|
4528
|
-
const buffer = await response.arrayBuffer();
|
|
4529
|
-
await writeFile(task.path, Buffer.from(buffer));
|
|
4530
|
-
task.log("Download successful");
|
|
4531
4670
|
};
|
|
4532
4671
|
var extractGtfsFiles = async (task) => {
|
|
4533
4672
|
if (!task.path) {
|
|
4534
|
-
throw new
|
|
4673
|
+
throw new GtfsError("No `path` specified in config", {
|
|
4674
|
+
code: "GTFS_CONFIG_INVALID" /* GTFS_CONFIG_INVALID */,
|
|
4675
|
+
category: "config" /* CONFIG */,
|
|
4676
|
+
details: { field: "path" }
|
|
4677
|
+
});
|
|
4535
4678
|
}
|
|
4536
4679
|
const gtfsPath = untildify(task.path);
|
|
4537
4680
|
task.log(`Importing static GTFS from ${task.path}\r`);
|
|
@@ -4543,19 +4686,34 @@ var extractGtfsFiles = async (task) => {
|
|
|
4543
4686
|
const files = await readdir(task.downloadDir);
|
|
4544
4687
|
const folders = files.filter((filename) => !["__MACOSX"].includes(filename)).map((filename) => path2.join(task.downloadDir, filename)).filter((source) => lstatSync(source).isDirectory());
|
|
4545
4688
|
if (folders.length > 1) {
|
|
4546
|
-
throw new
|
|
4547
|
-
`More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory
|
|
4689
|
+
throw new GtfsError(
|
|
4690
|
+
`More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
|
|
4691
|
+
{
|
|
4692
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4693
|
+
category: "zip" /* ZIP */,
|
|
4694
|
+
details: { path: task.path, folderCount: folders.length }
|
|
4695
|
+
}
|
|
4548
4696
|
);
|
|
4549
4697
|
} else if (folders.length === 0) {
|
|
4550
|
-
throw new
|
|
4551
|
-
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory
|
|
4698
|
+
throw new GtfsError(
|
|
4699
|
+
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
|
|
4700
|
+
{
|
|
4701
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4702
|
+
category: "zip" /* ZIP */,
|
|
4703
|
+
details: { path: task.path }
|
|
4704
|
+
}
|
|
4552
4705
|
);
|
|
4553
4706
|
}
|
|
4554
4707
|
const subfolderName = folders[0];
|
|
4555
4708
|
const directoryTextFiles = await getTextFiles(subfolderName);
|
|
4556
4709
|
if (directoryTextFiles.length === 0) {
|
|
4557
|
-
throw new
|
|
4558
|
-
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory
|
|
4710
|
+
throw new GtfsError(
|
|
4711
|
+
`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
|
|
4712
|
+
{
|
|
4713
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4714
|
+
category: "zip" /* ZIP */,
|
|
4715
|
+
details: { path: task.path, subfolderName }
|
|
4716
|
+
}
|
|
4559
4717
|
);
|
|
4560
4718
|
}
|
|
4561
4719
|
await Promise.all(
|
|
@@ -4568,15 +4726,27 @@ var extractGtfsFiles = async (task) => {
|
|
|
4568
4726
|
);
|
|
4569
4727
|
}
|
|
4570
4728
|
} catch (error) {
|
|
4571
|
-
|
|
4572
|
-
|
|
4729
|
+
const wrappedError = toGtfsError(error, {
|
|
4730
|
+
message: `Unable to unzip file ${task.path}`,
|
|
4731
|
+
code: "GTFS_ZIP_INVALID" /* GTFS_ZIP_INVALID */,
|
|
4732
|
+
category: "zip" /* ZIP */,
|
|
4733
|
+
details: { path: task.path }
|
|
4734
|
+
});
|
|
4735
|
+
task.logError(formatGtfsError(wrappedError));
|
|
4736
|
+
throw wrappedError;
|
|
4573
4737
|
}
|
|
4574
4738
|
} else {
|
|
4575
4739
|
try {
|
|
4576
4740
|
await cp(gtfsPath, task.downloadDir, { recursive: true });
|
|
4577
|
-
} catch {
|
|
4578
|
-
throw new
|
|
4579
|
-
`Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files
|
|
4741
|
+
} catch (error) {
|
|
4742
|
+
throw new GtfsError(
|
|
4743
|
+
`Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`,
|
|
4744
|
+
{
|
|
4745
|
+
code: "GTFS_DOWNLOAD_FAILED" /* GTFS_DOWNLOAD_FAILED */,
|
|
4746
|
+
category: "download" /* DOWNLOAD */,
|
|
4747
|
+
details: { path: gtfsPath },
|
|
4748
|
+
cause: error
|
|
4749
|
+
}
|
|
4580
4750
|
);
|
|
4581
4751
|
}
|
|
4582
4752
|
}
|
|
@@ -4671,8 +4841,17 @@ var formatGtfsLine = (line, model, totalLineCount) => {
|
|
|
4671
4841
|
if (value === "" || value === void 0 || value === null) {
|
|
4672
4842
|
formattedLine[name] = null;
|
|
4673
4843
|
if (required) {
|
|
4674
|
-
throw new
|
|
4675
|
-
`Missing required value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}
|
|
4844
|
+
throw new GtfsError(
|
|
4845
|
+
`Missing required value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`,
|
|
4846
|
+
{
|
|
4847
|
+
code: "GTFS_REQUIRED_FIELD_MISSING" /* GTFS_REQUIRED_FIELD_MISSING */,
|
|
4848
|
+
category: "validation" /* VALIDATION */,
|
|
4849
|
+
details: {
|
|
4850
|
+
file: `${filenameBase}.${filenameExtension}`,
|
|
4851
|
+
line: lineNumber,
|
|
4852
|
+
column: name
|
|
4853
|
+
}
|
|
4854
|
+
}
|
|
4676
4855
|
);
|
|
4677
4856
|
}
|
|
4678
4857
|
continue;
|
|
@@ -4680,8 +4859,18 @@ var formatGtfsLine = (line, model, totalLineCount) => {
|
|
|
4680
4859
|
if (type === "date") {
|
|
4681
4860
|
value = value?.toString().replace(/-/g, "");
|
|
4682
4861
|
if (value.length !== 8) {
|
|
4683
|
-
throw new
|
|
4684
|
-
`Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}
|
|
4862
|
+
throw new GtfsError(
|
|
4863
|
+
`Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`,
|
|
4864
|
+
{
|
|
4865
|
+
code: "GTFS_INVALID_DATE" /* GTFS_INVALID_DATE */,
|
|
4866
|
+
category: "validation" /* VALIDATION */,
|
|
4867
|
+
details: {
|
|
4868
|
+
file: `${filenameBase}.${filenameExtension}`,
|
|
4869
|
+
line: lineNumber,
|
|
4870
|
+
column: name,
|
|
4871
|
+
value
|
|
4872
|
+
}
|
|
4873
|
+
}
|
|
4685
4874
|
);
|
|
4686
4875
|
}
|
|
4687
4876
|
} else if (type === "time") {
|
|
@@ -4753,11 +4942,32 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4753
4942
|
task.logWarning(
|
|
4754
4943
|
`Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
|
|
4755
4944
|
);
|
|
4945
|
+
if (task.report) {
|
|
4946
|
+
addImportWarning(task.report, {
|
|
4947
|
+
code: "GTFS_DUPLICATE_PRIMARY_KEY" /* GTFS_DUPLICATE_PRIMARY_KEY */,
|
|
4948
|
+
message: `Duplicate values for primary key found in ${filename}.`,
|
|
4949
|
+
details: {
|
|
4950
|
+
file: filename,
|
|
4951
|
+
line: Number(rowNumber) + 1,
|
|
4952
|
+
columns: primaryColumns.map((column) => column.name)
|
|
4953
|
+
}
|
|
4954
|
+
});
|
|
4955
|
+
}
|
|
4756
4956
|
}
|
|
4757
4957
|
task.logWarning(
|
|
4758
4958
|
`Check ${filename} for invalid data on line ${rowNumber + 1}.`
|
|
4759
4959
|
);
|
|
4760
|
-
throw error
|
|
4960
|
+
throw toGtfsError(error, {
|
|
4961
|
+
message: error instanceof Error ? error.message : String(error),
|
|
4962
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
4963
|
+
category: "database" /* DATABASE */,
|
|
4964
|
+
details: {
|
|
4965
|
+
file: filename,
|
|
4966
|
+
line: Number(rowNumber) + 1,
|
|
4967
|
+
sqlitePath: task.sqlitePath,
|
|
4968
|
+
dbCode: error.code
|
|
4969
|
+
}
|
|
4970
|
+
});
|
|
4761
4971
|
}
|
|
4762
4972
|
}
|
|
4763
4973
|
});
|
|
@@ -4786,12 +4996,20 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4786
4996
|
}
|
|
4787
4997
|
}
|
|
4788
4998
|
} catch (error) {
|
|
4999
|
+
const gtfsError = toGtfsError(error, {
|
|
5000
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5001
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5002
|
+
category: "parse" /* PARSE */,
|
|
5003
|
+
details: { file: filename }
|
|
5004
|
+
});
|
|
4789
5005
|
if (task.ignoreErrors) {
|
|
4790
|
-
|
|
4791
|
-
task.logError(
|
|
5006
|
+
reportTaskError(task, gtfsError);
|
|
5007
|
+
task.logError(
|
|
5008
|
+
`Error processing ${filename}: ${gtfsError.message}`
|
|
5009
|
+
);
|
|
4792
5010
|
resolve();
|
|
4793
5011
|
} else {
|
|
4794
|
-
reject(
|
|
5012
|
+
reject(gtfsError);
|
|
4795
5013
|
}
|
|
4796
5014
|
}
|
|
4797
5015
|
});
|
|
@@ -4801,15 +5019,21 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4801
5019
|
try {
|
|
4802
5020
|
insertLines(lines);
|
|
4803
5021
|
} catch (error) {
|
|
5022
|
+
const gtfsError = toGtfsError(error, {
|
|
5023
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5024
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
5025
|
+
category: "database" /* DATABASE */,
|
|
5026
|
+
details: { file: filename, sqlitePath: task.sqlitePath }
|
|
5027
|
+
});
|
|
4804
5028
|
if (task.ignoreErrors) {
|
|
4805
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
4806
5029
|
task.logError(
|
|
4807
|
-
`Error inserting data for ${filename}: ${
|
|
5030
|
+
`Error inserting data for ${filename}: ${gtfsError.message}`
|
|
4808
5031
|
);
|
|
5032
|
+
reportTaskError(task, gtfsError);
|
|
4809
5033
|
resolve();
|
|
4810
5034
|
return;
|
|
4811
5035
|
} else {
|
|
4812
|
-
reject(
|
|
5036
|
+
reject(gtfsError);
|
|
4813
5037
|
return;
|
|
4814
5038
|
}
|
|
4815
5039
|
}
|
|
@@ -4820,22 +5044,38 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4820
5044
|
);
|
|
4821
5045
|
resolve();
|
|
4822
5046
|
} catch (error) {
|
|
5047
|
+
const gtfsError = toGtfsError(error, {
|
|
5048
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5049
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
5050
|
+
category: "database" /* DATABASE */,
|
|
5051
|
+
details: { file: filename, sqlitePath: task.sqlitePath }
|
|
5052
|
+
});
|
|
4823
5053
|
if (task.ignoreErrors) {
|
|
4824
|
-
|
|
4825
|
-
|
|
5054
|
+
task.logError(
|
|
5055
|
+
`Error finalizing ${filename}: ${gtfsError.message}`
|
|
5056
|
+
);
|
|
5057
|
+
reportTaskError(task, gtfsError);
|
|
4826
5058
|
resolve();
|
|
4827
5059
|
} else {
|
|
4828
|
-
reject(
|
|
5060
|
+
reject(gtfsError);
|
|
4829
5061
|
}
|
|
4830
5062
|
}
|
|
4831
5063
|
});
|
|
4832
5064
|
parser.on("error", (error) => {
|
|
5065
|
+
const gtfsError = toGtfsError(error, {
|
|
5066
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5067
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5068
|
+
category: "parse" /* PARSE */,
|
|
5069
|
+
details: { file: filename }
|
|
5070
|
+
});
|
|
4833
5071
|
if (task.ignoreErrors) {
|
|
4834
|
-
|
|
4835
|
-
|
|
5072
|
+
task.logError(
|
|
5073
|
+
`Parser error for ${filename}: ${gtfsError.message}`
|
|
5074
|
+
);
|
|
5075
|
+
reportTaskError(task, gtfsError);
|
|
4836
5076
|
resolve();
|
|
4837
5077
|
} else {
|
|
4838
|
-
reject(
|
|
5078
|
+
reject(gtfsError);
|
|
4839
5079
|
}
|
|
4840
5080
|
});
|
|
4841
5081
|
createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
|
|
@@ -4844,10 +5084,24 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4844
5084
|
if (isValidJSON(data) === false) {
|
|
4845
5085
|
if (task.ignoreErrors) {
|
|
4846
5086
|
task.logError(`Invalid JSON in ${filename}`);
|
|
5087
|
+
reportTaskError(
|
|
5088
|
+
task,
|
|
5089
|
+
new GtfsError(`Invalid JSON in ${filename}`, {
|
|
5090
|
+
code: "GTFS_JSON_INVALID" /* GTFS_JSON_INVALID */,
|
|
5091
|
+
category: "parse" /* PARSE */,
|
|
5092
|
+
details: { file: filename }
|
|
5093
|
+
})
|
|
5094
|
+
);
|
|
4847
5095
|
resolve();
|
|
4848
5096
|
return;
|
|
4849
5097
|
} else {
|
|
4850
|
-
reject(
|
|
5098
|
+
reject(
|
|
5099
|
+
new GtfsError(`Invalid JSON in ${filename}`, {
|
|
5100
|
+
code: "GTFS_JSON_INVALID" /* GTFS_JSON_INVALID */,
|
|
5101
|
+
category: "parse" /* PARSE */,
|
|
5102
|
+
details: { file: filename }
|
|
5103
|
+
})
|
|
5104
|
+
);
|
|
4851
5105
|
return;
|
|
4852
5106
|
}
|
|
4853
5107
|
}
|
|
@@ -4865,23 +5119,37 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4865
5119
|
);
|
|
4866
5120
|
resolve();
|
|
4867
5121
|
} catch (error) {
|
|
5122
|
+
const gtfsError = toGtfsError(error, {
|
|
5123
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5124
|
+
code: "GTFS_DB_OPERATION_FAILED" /* GTFS_DB_OPERATION_FAILED */,
|
|
5125
|
+
category: "database" /* DATABASE */,
|
|
5126
|
+
details: { file: filename, sqlitePath: task.sqlitePath }
|
|
5127
|
+
});
|
|
4868
5128
|
if (task.ignoreErrors) {
|
|
4869
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
4870
5129
|
task.logError(
|
|
4871
|
-
`Error inserting data for ${filename}: ${
|
|
5130
|
+
`Error inserting data for ${filename}: ${gtfsError.message}`
|
|
4872
5131
|
);
|
|
5132
|
+
reportTaskError(task, gtfsError);
|
|
4873
5133
|
resolve();
|
|
4874
5134
|
} else {
|
|
4875
|
-
reject(
|
|
5135
|
+
reject(gtfsError);
|
|
4876
5136
|
}
|
|
4877
5137
|
}
|
|
4878
5138
|
}).catch((error) => {
|
|
5139
|
+
const gtfsError = toGtfsError(error, {
|
|
5140
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5141
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5142
|
+
category: "parse" /* PARSE */,
|
|
5143
|
+
details: { file: filename }
|
|
5144
|
+
});
|
|
4879
5145
|
if (task.ignoreErrors) {
|
|
4880
|
-
|
|
4881
|
-
|
|
5146
|
+
task.logError(
|
|
5147
|
+
`Error reading ${filename}: ${gtfsError.message}`
|
|
5148
|
+
);
|
|
5149
|
+
reportTaskError(task, gtfsError);
|
|
4882
5150
|
resolve();
|
|
4883
5151
|
} else {
|
|
4884
|
-
reject(
|
|
5152
|
+
reject(gtfsError);
|
|
4885
5153
|
}
|
|
4886
5154
|
});
|
|
4887
5155
|
} else {
|
|
@@ -4892,7 +5160,17 @@ var importGtfsFiles = async (db, task) => {
|
|
|
4892
5160
|
resolve();
|
|
4893
5161
|
} else {
|
|
4894
5162
|
reject(
|
|
4895
|
-
new
|
|
5163
|
+
new GtfsError(
|
|
5164
|
+
`Unsupported file type: ${model.filenameExtension}`,
|
|
5165
|
+
{
|
|
5166
|
+
code: "GTFS_UNSUPPORTED_FILE_TYPE" /* GTFS_UNSUPPORTED_FILE_TYPE */,
|
|
5167
|
+
category: "parse" /* PARSE */,
|
|
5168
|
+
details: {
|
|
5169
|
+
file: filename,
|
|
5170
|
+
extension: model.filenameExtension
|
|
5171
|
+
}
|
|
5172
|
+
}
|
|
5173
|
+
)
|
|
4896
5174
|
);
|
|
4897
5175
|
}
|
|
4898
5176
|
}
|
|
@@ -4904,6 +5182,7 @@ async function importGtfs(initialConfig) {
|
|
|
4904
5182
|
const startTime = process.hrtime.bigint();
|
|
4905
5183
|
const config = setDefaultConfig(initialConfig);
|
|
4906
5184
|
validateConfigForImport(config);
|
|
5185
|
+
const report = config.includeImportReport ? createImportReport() : void 0;
|
|
4907
5186
|
try {
|
|
4908
5187
|
const db = openDb(config);
|
|
4909
5188
|
const agencyCount = config.agencies.length;
|
|
@@ -4931,7 +5210,8 @@ async function importGtfs(initialConfig) {
|
|
|
4931
5210
|
currentTimestamp: Math.floor(Date.now() / 1e3),
|
|
4932
5211
|
log: log(config),
|
|
4933
5212
|
logWarning: logWarning(config),
|
|
4934
|
-
logError: logError(config)
|
|
5213
|
+
logError: logError(config),
|
|
5214
|
+
report
|
|
4935
5215
|
};
|
|
4936
5216
|
if ("url" in agency2) {
|
|
4937
5217
|
Object.assign(task, { url: agency2.url });
|
|
@@ -4946,11 +5226,18 @@ async function importGtfs(initialConfig) {
|
|
|
4946
5226
|
await updateGtfsRealtimeData(task);
|
|
4947
5227
|
await rm2(tempPath, { recursive: true });
|
|
4948
5228
|
} catch (error) {
|
|
5229
|
+
const wrappedError = toGtfsError(error, {
|
|
5230
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5231
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5232
|
+
category: "parse" /* PARSE */
|
|
5233
|
+
});
|
|
4949
5234
|
if (config.ignoreErrors) {
|
|
4950
|
-
|
|
4951
|
-
|
|
5235
|
+
logError(config)(formatGtfsError(wrappedError));
|
|
5236
|
+
if (report) {
|
|
5237
|
+
addImportError(report, wrappedError);
|
|
5238
|
+
}
|
|
4952
5239
|
} else {
|
|
4953
|
-
throw
|
|
5240
|
+
throw wrappedError;
|
|
4954
5241
|
}
|
|
4955
5242
|
}
|
|
4956
5243
|
});
|
|
@@ -4964,11 +5251,29 @@ async function importGtfs(initialConfig) {
|
|
|
4964
5251
|
);
|
|
4965
5252
|
} catch (error) {
|
|
4966
5253
|
if (error.code === "SQLITE_CANTOPEN") {
|
|
4967
|
-
|
|
4968
|
-
`Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json
|
|
5254
|
+
const dbOpenError = new GtfsError(
|
|
5255
|
+
`Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`,
|
|
5256
|
+
{
|
|
5257
|
+
code: "DB_OPEN_FAILED" /* DB_OPEN_FAILED */,
|
|
5258
|
+
category: "database" /* DATABASE */,
|
|
5259
|
+
details: {
|
|
5260
|
+
sqlitePath: config.sqlitePath,
|
|
5261
|
+
dbCode: error.code
|
|
5262
|
+
},
|
|
5263
|
+
cause: error
|
|
5264
|
+
}
|
|
4969
5265
|
);
|
|
5266
|
+
logError(config)(dbOpenError.message);
|
|
5267
|
+
throw dbOpenError;
|
|
4970
5268
|
}
|
|
4971
|
-
throw error
|
|
5269
|
+
throw toGtfsError(error, {
|
|
5270
|
+
message: error instanceof Error ? error.message : String(error),
|
|
5271
|
+
code: "GTFS_CSV_PARSE_FAILED" /* GTFS_CSV_PARSE_FAILED */,
|
|
5272
|
+
category: "parse" /* PARSE */
|
|
5273
|
+
});
|
|
5274
|
+
}
|
|
5275
|
+
if (report) {
|
|
5276
|
+
return report;
|
|
4972
5277
|
}
|
|
4973
5278
|
}
|
|
4974
5279
|
|