@forzalabs/remora 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/docs/Remora - Code Sets.md +268 -0
- package/docs/Remora - Consumers.md +2232 -0
- package/docs/Remora - Producers.md +1658 -0
- package/docs/Remora - Sources.md +1150 -0
- package/documentation/README.md +262 -130
- package/documentation/default_resources/mock_data.csv +2 -2
- package/index.js +2215 -1637
- package/json_schemas/consumer-schema.json +146 -1
- package/package.json +1 -1
- package/workers/ExecutorWorker.js +611 -62
|
@@ -290,6 +290,46 @@
|
|
|
290
290
|
"copyFrom": {
|
|
291
291
|
"type": "string",
|
|
292
292
|
"description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
|
|
293
|
+
},
|
|
294
|
+
"position": {
|
|
295
|
+
"type": "object",
|
|
296
|
+
"description": "'FIXED' output only, and required on every visible field of one: where this field sits on the positional line and how its value is filled out to that width. Column order is field order and the offsets are computed from the widths, so no column number is transcribed by hand. A value longer than 'width' fails the run rather than being cut.",
|
|
297
|
+
"additionalProperties": false,
|
|
298
|
+
"required": ["width"],
|
|
299
|
+
"properties": {
|
|
300
|
+
"width": {
|
|
301
|
+
"type": "integer",
|
|
302
|
+
"minimum": 1,
|
|
303
|
+
"description": "How many characters the field occupies on the line."
|
|
304
|
+
},
|
|
305
|
+
"align": {
|
|
306
|
+
"type": "string",
|
|
307
|
+
"enum": ["left", "right"],
|
|
308
|
+
"description": "Which end of the field the value sits at, the other end taking the padding. Defaults by the field's type: 'left' for a string, 'right' for a number."
|
|
309
|
+
},
|
|
310
|
+
"pad": {
|
|
311
|
+
"type": "string",
|
|
312
|
+
"minLength": 1,
|
|
313
|
+
"maxLength": 1,
|
|
314
|
+
"description": "The single character the field is filled out with. Defaults by the field's type: a space for a string, '0' for a number."
|
|
315
|
+
},
|
|
316
|
+
"nullFill": {
|
|
317
|
+
"type": "string",
|
|
318
|
+
"minLength": 1,
|
|
319
|
+
"maxLength": 1,
|
|
320
|
+
"description": "The single character the full width of a NULL field is filled with. Defaults to a space, not to 'pad': zero-filling an empty numeric field writes a real value where there was none. An empty string counts as a null here."
|
|
321
|
+
},
|
|
322
|
+
"overflow": {
|
|
323
|
+
"type": "string",
|
|
324
|
+
"enum": ["fail", "truncate"],
|
|
325
|
+
"description": "What a value longer than 'width' does. 'fail' (the default) stops the run naming the field, the value and both widths; 'truncate' cuts the value to 'width', for the layouts that genuinely specify truncation."
|
|
326
|
+
},
|
|
327
|
+
"start": {
|
|
328
|
+
"type": "integer",
|
|
329
|
+
"minimum": 1,
|
|
330
|
+
"description": "Optional 1-based column the field is expected to start at. Not a second source of truth — the offsets always come from the widths — but an assertion that compile recomputes and fails on, so a forgotten filler is caught where it is written."
|
|
331
|
+
}
|
|
332
|
+
}
|
|
293
333
|
}
|
|
294
334
|
},
|
|
295
335
|
"required": [
|
|
@@ -310,11 +350,12 @@
|
|
|
310
350
|
"enum": [
|
|
311
351
|
"API",
|
|
312
352
|
"CSV",
|
|
353
|
+
"FIXED",
|
|
313
354
|
"JSON",
|
|
314
355
|
"PARQUET",
|
|
315
356
|
"TXT"
|
|
316
357
|
],
|
|
317
|
-
"description": "The output format of the consumer. JSON/CSV/TXT/PARQUET export to a file destination; API exposes the result over the API. PARQUET writes a typed, SNAPPY-compressed columnar file whose schema comes from the consumer's dimension types."
|
|
358
|
+
"description": "The output format of the consumer. JSON/CSV/TXT/PARQUET/FIXED export to a file destination; API exposes the result over the API. PARQUET writes a typed, SNAPPY-compressed columnar file whose schema comes from the consumer's dimension types. FIXED writes a positional flat file — no delimiter, no quotes, no header — where each visible field declares its own 'position'."
|
|
318
359
|
},
|
|
319
360
|
"exportDestination": {
|
|
320
361
|
"type": "string",
|
|
@@ -336,6 +377,81 @@
|
|
|
336
377
|
"type": "boolean",
|
|
337
378
|
"description": "'JSON'/'API' only. Omits the fields whose value is null from each exported record instead of writing them as \"field\": null — worth it on a wide, sparse record. Records then legitimately differ in shape, so it is rejected on a positional format (CSV/TXT/PARQUET), and on a JSON output whose consumer also writes one, since every output is serialized from the same record."
|
|
338
379
|
},
|
|
380
|
+
"recordLength": {
|
|
381
|
+
"type": "integer",
|
|
382
|
+
"minimum": 1,
|
|
383
|
+
"description": "'FIXED' only, and not alongside 'layouts', where each layout declares its own. The declared width of one record, which compile checks the field widths sum to and the export checks every composed line against. It is the length of the record as written, not the length of the copybook it came from: the fields past the cut are simply not declared rather than padded to. Every record is exactly this long, including one whose last field is null — that field is written full width in 'nullFill' and the line is never cut short."
|
|
384
|
+
},
|
|
385
|
+
"defaults": {
|
|
386
|
+
"type": "object",
|
|
387
|
+
"description": "'FIXED' only. The fill rules every field of one type takes unless its own 'position' overrides them, keyed by the field's type. A wide layout declares 'numbers are right-aligned and zero-padded' once instead of on each of 90 fields.",
|
|
388
|
+
"additionalProperties": false,
|
|
389
|
+
"properties": {
|
|
390
|
+
"string": { "$ref": "#/definitions/fixedFieldFill" },
|
|
391
|
+
"number": { "$ref": "#/definitions/fixedFieldFill" },
|
|
392
|
+
"datetime": { "$ref": "#/definitions/fixedFieldFill" },
|
|
393
|
+
"boolean": { "$ref": "#/definitions/fixedFieldFill" }
|
|
394
|
+
}
|
|
395
|
+
},
|
|
396
|
+
"lineEnding": {
|
|
397
|
+
"type": "string",
|
|
398
|
+
"enum": ["lf", "crlf"],
|
|
399
|
+
"description": "'FIXED' only. What ends every record, the last one included. Defaults to 'lf'. A real positional feed picks one and holds it — the NASCO POD files are CRLF on every line. A file with no terminator at all is fixed record framing (RECFM=FB) and is not expressible here."
|
|
400
|
+
},
|
|
401
|
+
"layouts": {
|
|
402
|
+
"type": "object",
|
|
403
|
+
"description": "'FIXED' only. Several record layouts in one file, picked per record by the value of one field — the write-side mirror of a producer's 'recordSelector'. The records come out grouped, one contiguous block per value in 'order', keeping within each block the order they were read in. Rejected together with a top-level 'recordLength': each layout declares its own. Without it a FIXED output writes every visible field as one layout.",
|
|
404
|
+
"additionalProperties": false,
|
|
405
|
+
"required": ["discriminator", "order", "values"],
|
|
406
|
+
"properties": {
|
|
407
|
+
"discriminator": {
|
|
408
|
+
"type": "string",
|
|
409
|
+
"description": "The consumer field whose value picks the layout, by the name it is exported under (the alias where one is set). It does not have to be written by any of the layouts, though it usually is."
|
|
410
|
+
},
|
|
411
|
+
"order": {
|
|
412
|
+
"type": "array",
|
|
413
|
+
"description": "The discriminator values in the order their blocks are written. The complete list: a record carrying a value that is not here has no block to go in and fails the run.",
|
|
414
|
+
"items": { "type": "string" },
|
|
415
|
+
"minItems": 1,
|
|
416
|
+
"uniqueItems": true
|
|
417
|
+
},
|
|
418
|
+
"values": {
|
|
419
|
+
"type": "object",
|
|
420
|
+
"description": "The layout each discriminator value is written with. Every value in 'order' needs an entry.",
|
|
421
|
+
"minProperties": 1,
|
|
422
|
+
"additionalProperties": {
|
|
423
|
+
"type": "object",
|
|
424
|
+
"additionalProperties": false,
|
|
425
|
+
"required": ["fields"],
|
|
426
|
+
"properties": {
|
|
427
|
+
"fields": {
|
|
428
|
+
"type": "array",
|
|
429
|
+
"description": "The fields this layout writes, in column order, by their exported names. A subset of the consumer's visible fields: the consumer declares the union of every layout's fields and each layout names the ones its records carry.",
|
|
430
|
+
"items": { "type": "string" },
|
|
431
|
+
"minItems": 1,
|
|
432
|
+
"uniqueItems": true
|
|
433
|
+
},
|
|
434
|
+
"recordLength": {
|
|
435
|
+
"type": "integer",
|
|
436
|
+
"minimum": 1,
|
|
437
|
+
"description": "As the output's 'recordLength', for this layout alone. A multi-layout output has no single one."
|
|
438
|
+
},
|
|
439
|
+
"defaults": {
|
|
440
|
+
"type": "object",
|
|
441
|
+
"description": "As the output's 'defaults', applied over them for this layout's fields alone.",
|
|
442
|
+
"additionalProperties": false,
|
|
443
|
+
"properties": {
|
|
444
|
+
"string": { "$ref": "#/definitions/fixedFieldFill" },
|
|
445
|
+
"number": { "$ref": "#/definitions/fixedFieldFill" },
|
|
446
|
+
"datetime": { "$ref": "#/definitions/fixedFieldFill" },
|
|
447
|
+
"boolean": { "$ref": "#/definitions/fixedFieldFill" }
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
},
|
|
339
455
|
"trigger": {
|
|
340
456
|
"type": "object",
|
|
341
457
|
"description": "Triggers to perform the export (not just the usual 'Deploy')",
|
|
@@ -767,6 +883,35 @@
|
|
|
767
883
|
],
|
|
768
884
|
"additionalProperties": false,
|
|
769
885
|
"definitions": {
|
|
886
|
+
"fixedFieldFill": {
|
|
887
|
+
"type": "object",
|
|
888
|
+
"description": "How a value is filled out to the width of its fixed-width field, set once for every field of one type. A field's own 'position' wins where both set the same rule.",
|
|
889
|
+
"additionalProperties": false,
|
|
890
|
+
"properties": {
|
|
891
|
+
"align": {
|
|
892
|
+
"type": "string",
|
|
893
|
+
"enum": ["left", "right"],
|
|
894
|
+
"description": "Which end of the field the value sits at, the other end taking the padding. Defaults by the field's type: 'left' for a string, 'right' for a number."
|
|
895
|
+
},
|
|
896
|
+
"pad": {
|
|
897
|
+
"type": "string",
|
|
898
|
+
"minLength": 1,
|
|
899
|
+
"maxLength": 1,
|
|
900
|
+
"description": "The single character the field is filled out with. Defaults by the field's type: a space for a string, '0' for a number."
|
|
901
|
+
},
|
|
902
|
+
"nullFill": {
|
|
903
|
+
"type": "string",
|
|
904
|
+
"minLength": 1,
|
|
905
|
+
"maxLength": 1,
|
|
906
|
+
"description": "The single character the full width of a NULL field is filled with. Defaults to a space, not to 'pad': zero-filling an empty numeric field writes a real value where there was none. An empty string counts as a null here."
|
|
907
|
+
},
|
|
908
|
+
"overflow": {
|
|
909
|
+
"type": "string",
|
|
910
|
+
"enum": ["fail", "truncate"],
|
|
911
|
+
"description": "What a value longer than 'width' does. 'fail' (the default) stops the run naming the field, the value and both widths; 'truncate' cuts the value to 'width', for the layouts that genuinely specify truncation."
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
},
|
|
770
915
|
"singleTransformation": {
|
|
771
916
|
"oneOf": [
|
|
772
917
|
{
|