@golemio/public-transport 0.1.0-dev.2780201953 → 0.1.0-dev.2780325741
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -32,6 +32,21 @@ let GtfsTransformation = class GtfsTransformation extends AbstractTransformation
|
|
|
32
32
|
return acc;
|
|
33
33
|
}, {});
|
|
34
34
|
const { feedId } = input;
|
|
35
|
+
// some feed exports (e.g. IDS JMK) prepend a UTF-8 BOM to every file; left in place it
|
|
36
|
+
// sticks to the first header name (e.g. "stop_id"), which then fails to match any
|
|
37
|
+
// known column and gets dropped, nulling out the whole first column (often the PK).
|
|
38
|
+
let bomChecked = false;
|
|
39
|
+
const stripBom = new stream_1.Transform({
|
|
40
|
+
transform(chunk, _encoding, callback) {
|
|
41
|
+
if (!bomChecked) {
|
|
42
|
+
bomChecked = true;
|
|
43
|
+
if (chunk.length >= 3 && chunk[0] === 0xef && chunk[1] === 0xbb && chunk[2] === 0xbf) {
|
|
44
|
+
chunk = chunk.subarray(3);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
callback(null, chunk);
|
|
48
|
+
},
|
|
49
|
+
});
|
|
35
50
|
const addColumns = new stream_1.Transform({
|
|
36
51
|
objectMode: true,
|
|
37
52
|
transform(chunk, _encoding, callback) {
|
|
@@ -51,7 +66,7 @@ let GtfsTransformation = class GtfsTransformation extends AbstractTransformation
|
|
|
51
66
|
callback();
|
|
52
67
|
},
|
|
53
68
|
});
|
|
54
|
-
const out = (0, stream_1.pipeline)(input.sourceStream, (0, csv_parser_1.default)(), addColumns, (0, csv_stringify_1.default)({ header: true }), (err) => {
|
|
69
|
+
const out = (0, stream_1.pipeline)(input.sourceStream, stripBom, (0, csv_parser_1.default)(), addColumns, (0, csv_stringify_1.default)({ header: true }), (err) => {
|
|
55
70
|
if (err) {
|
|
56
71
|
integration_engine_1.log.error(`Datasets: transformation error (${input.name}): ${err.toString()}`);
|
|
57
72
|
throw err;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GtfsTransformation.js","sourceRoot":"","sources":["../../../../src/integration-engine/gtfs/transformations/GtfsTransformation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2DAAiC;AAEjC,8EAA4D;AAC5D,6GAA0G;AAC1G,iEAAgE;AAChE,4DAA6B;AAC7B,kEAAsC;AACtC,mCAAuD;AAShD,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,+CAAyD;IAG7F;QACI,KAAK,EAAE,CAAC;QAIF,sBAAiB,GAAG,CAAC,KAA8B,EAAY,EAAE;YACvE,iEAAiE;YACjE,MAAM,4BAA4B,GAAG,MAAM,CAAC,IAAI,CAAC,WAAI,CAAC,KAAK,CAAC,IAAkB,CAAC,CAAC,yBAAyB,CAAC;iBACrG,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;iBACnD,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBAClB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;gBACjB,OAAO,GAAG,CAAC;YACf,CAAC,EAAE,EAA0B,CAAC,CAAC;YAEnC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;YAEzB,MAAM,UAAU,GAAG,IAAI,kBAAS,CAAC;gBAC7B,UAAU,EAAE,IAAI;gBAChB,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;oBAChC,IAAI,CAAC,IAAI,CAAC;wBACN,sEAAsE;wBACtE,GAAG,4BAA4B;wBAC/B,kEAAkE;wBAClE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;6BAChB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,4BAA4B,CAAC;6BACpD,MAAM,CAAC,CAAC,GAAQ,EAAE,GAAG,EAAE,EAAE;4BACtB,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;4BACtB,OAAO,GAAG,CAAC;wBACf,CAAC,EAAE,EAAE,CAAC;wBACV,qEAAqE;wBACrE,OAAO,EAAE,MAAM;qBAClB,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;gBACf,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,GAAG,GAAG,IAAA,iBAAQ,EAAC,KAAK,CAAC,YAAY,EAAE,IAAA,oBAAG,GAAE,EAAE,UAAU,EAAE,IAAA,uBAAS,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE;
|
|
1
|
+
{"version":3,"file":"GtfsTransformation.js","sourceRoot":"","sources":["../../../../src/integration-engine/gtfs/transformations/GtfsTransformation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2DAAiC;AAEjC,8EAA4D;AAC5D,6GAA0G;AAC1G,iEAAgE;AAChE,4DAA6B;AAC7B,kEAAsC;AACtC,mCAAuD;AAShD,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,+CAAyD;IAG7F;QACI,KAAK,EAAE,CAAC;QAIF,sBAAiB,GAAG,CAAC,KAA8B,EAAY,EAAE;YACvE,iEAAiE;YACjE,MAAM,4BAA4B,GAAG,MAAM,CAAC,IAAI,CAAC,WAAI,CAAC,KAAK,CAAC,IAAkB,CAAC,CAAC,yBAAyB,CAAC;iBACrG,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;iBACnD,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBAClB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;gBACjB,OAAO,GAAG,CAAC;YACf,CAAC,EAAE,EAA0B,CAAC,CAAC;YAEnC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;YAEzB,uFAAuF;YACvF,mFAAmF;YACnF,oFAAoF;YACpF,IAAI,UAAU,GAAG,KAAK,CAAC;YACvB,MAAM,QAAQ,GAAG,IAAI,kBAAS,CAAC;gBAC3B,SAAS,CAAC,KAAa,EAAE,SAAS,EAAE,QAAQ;oBACxC,IAAI,CAAC,UAAU,EAAE,CAAC;wBACd,UAAU,GAAG,IAAI,CAAC;wBAClB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;4BACnF,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC9B,CAAC;oBACL,CAAC;oBACD,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC1B,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,kBAAS,CAAC;gBAC7B,UAAU,EAAE,IAAI;gBAChB,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;oBAChC,IAAI,CAAC,IAAI,CAAC;wBACN,sEAAsE;wBACtE,GAAG,4BAA4B;wBAC/B,kEAAkE;wBAClE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;6BAChB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,4BAA4B,CAAC;6BACpD,MAAM,CAAC,CAAC,GAAQ,EAAE,GAAG,EAAE,EAAE;4BACtB,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;4BACtB,OAAO,GAAG,CAAC;wBACf,CAAC,EAAE,EAAE,CAAC;wBACV,qEAAqE;wBACrE,OAAO,EAAE,MAAM;qBAClB,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;gBACf,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,GAAG,GAAG,IAAA,iBAAQ,EAAC,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,IAAA,oBAAG,GAAE,EAAE,UAAU,EAAE,IAAA,uBAAS,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE;gBACvG,IAAI,GAAG,EAAE,CAAC;oBACN,wBAAG,CAAC,KAAK,CAAC,mCAAmC,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC/E,MAAM,GAAG,CAAC;gBACd,CAAC;qBAAM,CAAC;oBACJ,wBAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;gBACjE,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,OAAO,GAAG,CAAC;QACf,CAAC,CAAC;QA5DE,IAAI,CAAC,IAAI,GAAG,WAAI,CAAC,IAAI,CAAC;IAC1B,CAAC;CA4DJ,CAAA;AAlEY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,qBAAU,GAAE;;GACA,kBAAkB,CAkE9B"}
|