@forzalabs/remora 2.5.0 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENCE.md +52 -17
- package/README.md +11 -1
- package/RELEASE_NOTES.md +157 -0
- package/docs/Remora - Code Sets.md +1 -1
- package/docs/Remora - Consumers.md +19 -11
- package/index.js +43 -19
- package/json_schemas/consumer-schema.json +1 -1
- package/package.json +2 -2
- package/workers/ExecutorWorker.js +43 -19
- package/CHANGELOG.md +0 -537
package/LICENCE.md
CHANGED
|
@@ -1,31 +1,66 @@
|
|
|
1
|
-
|
|
1
|
+
# Remora Licence
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Copyright (c) 2026 Forza Labs LLC. All rights reserved.
|
|
4
4
|
|
|
5
|
-
Licensor
|
|
5
|
+
**Licensor:** Forza Labs LLC
|
|
6
6
|
|
|
7
|
-
Licensed Work
|
|
7
|
+
**Licensed Work:** Remora, including the `@forzalabs/remora` and `@forzalabs/remora-preview`
|
|
8
|
+
packages and the documentation and schemas distributed with them.
|
|
8
9
|
|
|
9
|
-
|
|
10
|
+
This is a proprietary licence. It is **not** an open source licence, and it does not convert
|
|
11
|
+
into one. The Licensed Work is made available on the terms below and on no others.
|
|
10
12
|
|
|
11
|
-
|
|
13
|
+
## 1. Non-commercial use
|
|
12
14
|
|
|
13
|
-
|
|
15
|
+
You may use, copy, modify and redistribute the Licensed Work free of charge for any
|
|
16
|
+
non-commercial purpose, including personal, academic, research, evaluation and open source
|
|
17
|
+
projects, provided you comply with sections 3 to 7.
|
|
14
18
|
|
|
15
|
-
|
|
19
|
+
Use is non-commercial where it is not carried out for, on behalf of, or for the benefit of a
|
|
20
|
+
company or other organisation, and is not directed towards commercial advantage or monetary
|
|
21
|
+
compensation.
|
|
16
22
|
|
|
17
|
-
|
|
23
|
+
## 2. Commercial use
|
|
18
24
|
|
|
19
|
-
|
|
25
|
+
Any other use requires a separate written commercial licence from the Licensor. This includes
|
|
26
|
+
any use for, on behalf of, or for the benefit of a company or other organisation, and any use
|
|
27
|
+
directed towards commercial advantage or monetary compensation.
|
|
20
28
|
|
|
21
|
-
|
|
29
|
+
Without such a licence, no right to use the Licensed Work for those purposes is granted. To
|
|
30
|
+
obtain one, contact admin@forza-labs.com.
|
|
22
31
|
|
|
23
|
-
|
|
24
|
-
- This license does not grant trademark rights.
|
|
25
|
-
- You must include this license text with any distribution of the Licensed Work.
|
|
26
|
-
- You may not remove or modify this license notice.
|
|
32
|
+
## 3. Separate agreements take precedence
|
|
27
33
|
|
|
28
|
-
|
|
34
|
+
Where you have entered into a separate written agreement with the Licensor covering the
|
|
35
|
+
Licensed Work, that agreement governs your use of it and supersedes this notice in full to the
|
|
36
|
+
extent of any conflict.
|
|
29
37
|
|
|
30
|
-
|
|
38
|
+
## 4. Notices
|
|
31
39
|
|
|
40
|
+
You must include this licence text, unmodified, with any copy or redistribution of the Licensed
|
|
41
|
+
Work. You may not remove, alter or obscure this notice, or any copyright, licence or
|
|
42
|
+
attribution notice contained in the Licensed Work.
|
|
43
|
+
|
|
44
|
+
## 5. Trademarks
|
|
45
|
+
|
|
46
|
+
This licence grants no right to use the names, logos or trademarks of the Licensor, including
|
|
47
|
+
"Forza Labs" and "Remora", beyond the attribution section 4 requires.
|
|
48
|
+
|
|
49
|
+
## 6. No warranty and limitation of liability
|
|
50
|
+
|
|
51
|
+
THE LICENSED WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
52
|
+
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
|
53
|
+
PURPOSE, TITLE AND NONINFRINGEMENT. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO
|
|
54
|
+
EVENT SHALL THE LICENSOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
55
|
+
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED
|
|
56
|
+
WORK OR THE USE OF OR OTHER DEALINGS IN IT.
|
|
57
|
+
|
|
58
|
+
## 7. Termination
|
|
59
|
+
|
|
60
|
+
If you use the Licensed Work other than as this notice permits, the rights granted to you end
|
|
61
|
+
automatically. They are reinstated only once you cease the non-compliant use and the Licensor
|
|
62
|
+
confirms reinstatement in writing.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
Commercial licensing enquiries: admin@forza-labs.com
|
package/README.md
CHANGED
|
@@ -39,9 +39,19 @@ remora compile
|
|
|
39
39
|
remora deploy
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
+
## 📋 Release notes
|
|
43
|
+
|
|
44
|
+
What changed in each release, and what to check before upgrading, is in
|
|
45
|
+
[RELEASE_NOTES.md](RELEASE_NOTES.md), which ships with this package.
|
|
46
|
+
|
|
42
47
|
## 📄 License
|
|
43
48
|
|
|
44
|
-
|
|
49
|
+
Proprietary. Free for non-commercial use — personal, academic, research, evaluation and open source
|
|
50
|
+
projects. Any use for, on behalf of, or for the benefit of a company or other organisation requires a
|
|
51
|
+
separate written commercial licence.
|
|
52
|
+
|
|
53
|
+
The full terms are in [LICENCE.md](LICENCE.md), which ships with this package. For commercial
|
|
54
|
+
licensing, contact admin@forza-labs.com.
|
|
45
55
|
|
|
46
56
|
---
|
|
47
57
|
|
package/RELEASE_NOTES.md
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Remora release notes
|
|
2
|
+
|
|
3
|
+
What changed in each release, and what to check before upgrading. Newest first.
|
|
4
|
+
|
|
5
|
+
Run `remora -v` to see the version you have installed.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Unreleased
|
|
10
|
+
|
|
11
|
+
## 2.6.1 — 2026-10-27
|
|
12
|
+
|
|
13
|
+
**Changed**
|
|
14
|
+
|
|
15
|
+
- Remora is distributed under a restated proprietary licence. What you may do is unchanged: free for
|
|
16
|
+
personal, academic, research, evaluation and open source use, with a written commercial licence
|
|
17
|
+
required for use by or on behalf of an organisation. Full text in `LICENCE.md`. `remora ui-preview`
|
|
18
|
+
carries the same licence and now states it in the package.
|
|
19
|
+
- These release notes ship inside the package, so the record of what changed in the version you have
|
|
20
|
+
installed sits on disk beside it. The development changelog, which was included up to 2.6.0, is an
|
|
21
|
+
internal working record and is no longer part of the package.
|
|
22
|
+
|
|
23
|
+
**Upgrading.** Nothing in a project needs changing. If anything of yours reads
|
|
24
|
+
`node_modules/@forzalabs/remora/CHANGELOG.md`, point it at `RELEASE_NOTES.md` in the same folder.
|
|
25
|
+
|
|
26
|
+
## 2.6.0 — 2026-10-26
|
|
27
|
+
|
|
28
|
+
**Changed**
|
|
29
|
+
|
|
30
|
+
- A `CSV` or `TXT` export quotes a field only where the quotes carry meaning — when the value holds the
|
|
31
|
+
delimiter, a `"`, a line break, or a space at either end. An empty string stays quoted, so it is still
|
|
32
|
+
distinct from a null. Exported files are around 15% smaller.
|
|
33
|
+
|
|
34
|
+
**Upgrading.** Nothing in a project needs changing. Check any destination that compares exported files
|
|
35
|
+
against a fixture, or that parses them by splitting on `","` rather than with a CSV reader.
|
|
36
|
+
|
|
37
|
+
## 2.5.0 — 2026-10-26
|
|
38
|
+
|
|
39
|
+
**Added**
|
|
40
|
+
|
|
41
|
+
- `"format": "FIXED"`, a positional flat-file output: no delimiter, no quotes, no header. Each field
|
|
42
|
+
declares a `width`, with `align`, `pad`, `nullFill` and `overflow` controlling how it is filled.
|
|
43
|
+
- `recordLength`, `defaults` and `lineEnding` on the output, and `layouts` for several record layouts in
|
|
44
|
+
one file. Column offsets are computed from the widths, so none are written by hand.
|
|
45
|
+
- The full prose reference now ships with the CLI, so the documentation always matches your version.
|
|
46
|
+
|
|
47
|
+
**Upgrading.** Nothing changes unless a consumer declares a `FIXED` output. A FIXED file leaves Remora as
|
|
48
|
+
UTF-8 text, so a z/OS target still needs the ASCII→EBCDIC and `RECFM=FB` conversion on its own side.
|
|
49
|
+
|
|
50
|
+
## 2.4.0 — 2026-10-19
|
|
51
|
+
|
|
52
|
+
**Added**
|
|
53
|
+
|
|
54
|
+
- `"format": "PARQUET"`, a fourth file output.
|
|
55
|
+
|
|
56
|
+
**Changed**
|
|
57
|
+
|
|
58
|
+
- A `CSV` or `TXT` export distinguishes a null from an empty string.
|
|
59
|
+
- A value containing a newline no longer becomes two records.
|
|
60
|
+
- A consumer declares exactly one output.
|
|
61
|
+
|
|
62
|
+
**Removed**
|
|
63
|
+
|
|
64
|
+
- `measures` on a producer. It never calculated anything, so a consumer naming one exported an empty
|
|
65
|
+
column.
|
|
66
|
+
|
|
67
|
+
**Fixed**
|
|
68
|
+
|
|
69
|
+
- A `hidden` field no longer shifts the columns after it.
|
|
70
|
+
- A `pivot` now tells downstream passes that the shape changed, and runs on sources under 10 MB.
|
|
71
|
+
- A boolean read from a text source is no longer always `true`.
|
|
72
|
+
- A `lineCount` field is renumbered against the file it belongs to.
|
|
73
|
+
- A clean exit no longer deletes a concurrent run's temporary folder.
|
|
74
|
+
|
|
75
|
+
**Upgrading.** A producer carrying `measures` is refused at `compile` — delete the key, including an empty
|
|
76
|
+
`"measures": []`. Run `remora compile` before upgrading in place.
|
|
77
|
+
|
|
78
|
+
## 2.3.0 — 2026-10-15
|
|
79
|
+
|
|
80
|
+
**Added**
|
|
81
|
+
|
|
82
|
+
- `remora report [run-id]` packages a run — versions, redacted configuration, the run record and its slice
|
|
83
|
+
of the log — into a `.zip` you can send to someone else.
|
|
84
|
+
|
|
85
|
+
**Changed**
|
|
86
|
+
|
|
87
|
+
- A `CSV` or `TXT` export writes a header line naming its columns.
|
|
88
|
+
- `remora package` and `remora deploy` refuse an archive containing a `.env` file.
|
|
89
|
+
|
|
90
|
+
**Upgrading.** Every exported `CSV` and `TXT` file gains a first line. Check any receiver that expects data
|
|
91
|
+
on line one.
|
|
92
|
+
|
|
93
|
+
## 2.2.0 / 2.2.1 — 2026-10-15
|
|
94
|
+
|
|
95
|
+
**Added**
|
|
96
|
+
|
|
97
|
+
- `split` on a consumer: the output broken into several files by a field value, a size cap or a record
|
|
98
|
+
count.
|
|
99
|
+
- `remora discover <producer> --replace`.
|
|
100
|
+
- A consumer's filters and the run's record limit reach a SQL source as a `WHERE` and a `LIMIT`.
|
|
101
|
+
- A `presigned-url` engine, usable as a source and as a destination.
|
|
102
|
+
|
|
103
|
+
**Changed**
|
|
104
|
+
|
|
105
|
+
- `nullable` on a producer dimension decides what a missing value becomes. It defaults to `true`, which
|
|
106
|
+
keeps a missing value as null.
|
|
107
|
+
- The configuration archive is called `remora.zip`.
|
|
108
|
+
- `settings.sql` on a producer, and the resource folder names in `project.json`, are removed.
|
|
109
|
+
|
|
110
|
+
**Fixed**
|
|
111
|
+
|
|
112
|
+
- `remora discover` no longer overwrites hand-authored dimension properties, and reports what it found on
|
|
113
|
+
a SQL source.
|
|
114
|
+
- A NULL in a `number` column is no longer staged as `0`.
|
|
115
|
+
- An export no longer loses its last row, and a failed export no longer finishes the run as a success.
|
|
116
|
+
- `region` and `bucket` can be `{ENV_VAR}` references on an S3 or Redshift source.
|
|
117
|
+
|
|
118
|
+
**Upgrading.** `nullable` now affects what a run produces — a `number` column holding NULLs stops emitting
|
|
119
|
+
`0`. Remove `settings.sql` and the folder names from existing configurations.
|
|
120
|
+
|
|
121
|
+
## 2.1.0 / 2.1.1 — 2026-09
|
|
122
|
+
|
|
123
|
+
**Added**
|
|
124
|
+
|
|
125
|
+
- SQL sources can be run directly.
|
|
126
|
+
- Postgres authenticates with an RDS IAM auth token.
|
|
127
|
+
|
|
128
|
+
**Changed**
|
|
129
|
+
|
|
130
|
+
- A live run reports what it read without needing `REMORA_DEBUG_MODE`.
|
|
131
|
+
|
|
132
|
+
**Fixed**
|
|
133
|
+
|
|
134
|
+
- A delta-share producer no longer exhausts worker memory.
|
|
135
|
+
- A ZIP producer no longer reads the whole archive into memory.
|
|
136
|
+
- A multi-line error arrives as one log event rather than many unrelated ones.
|
|
137
|
+
|
|
138
|
+
## 2.0.0 — 2026-09-01
|
|
139
|
+
|
|
140
|
+
A performance release. A run shares one worker pool across every producer, file and chunk, per-record work
|
|
141
|
+
and output serialization are compiled once rather than per record, and workers write in batches.
|
|
142
|
+
|
|
143
|
+
**Fixed**
|
|
144
|
+
|
|
145
|
+
- JSON schema validation runs when a project loads.
|
|
146
|
+
- A filter on a `number`, `datetime` or `boolean` field matches.
|
|
147
|
+
- A run's record limit applies to the whole run, not to each worker.
|
|
148
|
+
- A masked producer dimension is no longer shown in the clear when sampled (2.0.3).
|
|
149
|
+
|
|
150
|
+
**Upgrading.** A configuration that was silently schema-invalid now fails at `compile`. Run `remora compile`
|
|
151
|
+
before upgrading in place.
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
Earlier 1.x releases are superseded by 2.0. Questions and bug reports:
|
|
156
|
+
[@forzalabs/remora](https://www.npmjs.com/package/@forzalabs/remora) — `remora report` packages a run worth
|
|
157
|
+
attaching.
|
|
@@ -177,7 +177,7 @@ Remora ships built-in healthcare/payer-quality code sets that are applied **auto
|
|
|
177
177
|
|
|
178
178
|
Because `measure_code` and `measure_name` both bind to the built-in `hedis_measures` set, they are automatically correlated on each row — with no configuration.
|
|
179
179
|
|
|
180
|
-
A built-in set is only applied when it can produce the dimension's declared `type`. A numeric surrogate key such as `
|
|
180
|
+
A built-in set is only applied when it can produce the dimension's declared `type`. A numeric surrogate key such as `quality_gaps_measure_code_sk` matches `measure_code` by name, but HEDIS codes are not numbers, so the binding is skipped and the field is generated from its own type instead. An **explicit** `synth.codeSet` or `synth.allowedValues` that cannot produce the declared type is a configuration error and fails the run by name.
|
|
181
181
|
|
|
182
182
|
### **Overriding a built-in set**
|
|
183
183
|
|
|
@@ -1042,11 +1042,11 @@ Records then legitimately differ in shape from one another, which is why it appl
|
|
|
1042
1042
|
|
|
1043
1043
|
### **The Header Line**
|
|
1044
1044
|
|
|
1045
|
-
A `CSV` or `TXT` file is positional: it carries values and nothing that names them. So both formats write a **header line** first — the exported field names, in column order, quoted and delimited exactly
|
|
1045
|
+
A `CSV` or `TXT` file is positional: it carries values and nothing that names them. So both formats write a **header line** first — the exported field names, in column order, quoted and delimited on exactly the rule the rows under it follow, so the header parses with the same reader the data does:
|
|
1046
1046
|
|
|
1047
1047
|
```
|
|
1048
|
-
|
|
1049
|
-
|
|
1048
|
+
id,primary_email,plan,country
|
|
1049
|
+
USR-00001,alice.rossi@example.com,free,Italy
|
|
1050
1050
|
```
|
|
1051
1051
|
|
|
1052
1052
|
The names are the exported ones: a field's `alias` where it has one, its `key` otherwise. A `hidden` field is not written, so it is not in the header either. With a `split`, **every** file of the output carries the header, not just the first one — each file has to stand on its own.
|
|
@@ -1070,20 +1070,28 @@ It only applies to `CSV` and `TXT`. On a `JSON` or `API` output each record alre
|
|
|
1070
1070
|
|
|
1071
1071
|
### **How a value is written**
|
|
1072
1072
|
|
|
1073
|
-
A `CSV`/`TXT` file
|
|
1074
|
-
|
|
1075
|
-
|
|
1073
|
+
A `CSV`/`TXT` file **quotes a value only where the quotes carry meaning**. A value is written bare
|
|
1074
|
+
unless it holds the delimiter, a `"`, a line break, or a space at either end — and an empty string
|
|
1075
|
+
is quoted too, because that is what leaves the unquoted empty field free to mean something else: it
|
|
1076
|
+
is a **null**. The two are different values, and on a positional line this is the only way to tell
|
|
1077
|
+
them apart:
|
|
1076
1078
|
|
|
1077
1079
|
```
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1080
|
+
id,note,amount
|
|
1081
|
+
USR-1,"",12.5 <- note is the empty string
|
|
1082
|
+
USR-2,, <- note and amount are null
|
|
1083
|
+
USR-3,"a, b",12.5 <- note holds the delimiter, so it is quoted
|
|
1084
|
+
USR-4," padded ",0 <- note has edge spaces, so it is quoted
|
|
1081
1085
|
```
|
|
1082
1086
|
|
|
1083
1087
|
A reader that treats an unquoted empty field as "no value" — which most CSV readers do, and which is
|
|
1084
1088
|
what `COPY`, `LOAD DATA` and a pandas `read_csv` all assume — gets a null back. A reader that does
|
|
1085
1089
|
not sees an empty string, exactly as before.
|
|
1086
1090
|
|
|
1091
|
+
Edge spaces are quoted for the same reason the empty string is: most CSV readers, Remora's own
|
|
1092
|
+
included, trim an unquoted field, so ` padded ` written bare would come back shorter than it went
|
|
1093
|
+
out. Quoting it is what makes the round trip exact.
|
|
1094
|
+
|
|
1087
1095
|
A value holding a **newline** or a carriage return is written as the two characters `\n` or `\r`.
|
|
1088
1096
|
A line of a remora file is one record, everywhere in the pipeline, so a value carrying a real
|
|
1089
1097
|
newline would otherwise arrive as two broken rows. The escape is one-way: nothing turns it back into
|
|
@@ -1263,7 +1271,7 @@ How it behaves:
|
|
|
1263
1271
|
|
|
1264
1272
|
### **How a Record Ends**
|
|
1265
1273
|
|
|
1266
|
-
`lineEnding` is `"lf"` (the default) or `"crlf"`, and it ends **every** record, the last one included. A real positional feed picks one and holds it:
|
|
1274
|
+
`lineEnding` is `"lf"` (the default) or `"crlf"`, and it ends **every** record, the last one included. A real positional feed picks one and holds it: a POD-style transaction feed, for instance, may be CRLF throughout.
|
|
1267
1275
|
|
|
1268
1276
|
A file with **no terminator at all** — fixed record framing, `RECFM=FB`, where the record length alone says where one record ends — is not expressible here, and asking for it is the signal that what is needed is byte-mode framing rather than a line-based file. Remora does not write one.
|
|
1269
1277
|
|
|
@@ -1307,7 +1315,7 @@ Apache Parquet: a typed, SNAPPY-compressed **columnar** file, which is what a da
|
|
|
1307
1315
|
}
|
|
1308
1316
|
```
|
|
1309
1317
|
|
|
1310
|
-
Two things it gives a reader that a `CSV` export cannot. The file **declares its types**, so nothing downstream has to guess them back out of the text. And it is much smaller: a four-column, 2-million-row dataset is about 28 MB as parquet against
|
|
1318
|
+
Two things it gives a reader that a `CSV` export cannot. The file **declares its types**, so nothing downstream has to guess them back out of the text. And it is much smaller: a four-column, 2-million-row dataset is about 28 MB as parquet against roughly 100 MB as CSV.
|
|
1311
1319
|
|
|
1312
1320
|
The schema comes from what the consumer already declares — each field's dimension type:
|
|
1313
1321
|
|
package/index.js
CHANGED
|
@@ -11761,7 +11761,7 @@ var CONSTANTS, Constants_default;
|
|
|
11761
11761
|
var init_Constants = __esm({
|
|
11762
11762
|
"../../packages/constants/src/Constants.ts"() {
|
|
11763
11763
|
CONSTANTS = {
|
|
11764
|
-
cliVersion: "2.
|
|
11764
|
+
cliVersion: "2.6.1",
|
|
11765
11765
|
backendVersion: 1,
|
|
11766
11766
|
backendPort: 5088,
|
|
11767
11767
|
workerVersion: 2,
|
|
@@ -11889,7 +11889,7 @@ var init_Constants = __esm({
|
|
|
11889
11889
|
/**
|
|
11890
11890
|
* Record layouts one FIXED output may write into a single file, which is also how many write
|
|
11891
11891
|
* streams its block-ordering pass holds open. A copybook of this size is already unusual —
|
|
11892
|
-
*
|
|
11892
|
+
* a POD-style transaction feed may write five transaction types — and the bound stops a generated config
|
|
11893
11893
|
* from opening a descriptor per distinct value of the wrong field.
|
|
11894
11894
|
*/
|
|
11895
11895
|
MAX_FIXED_LAYOUTS: 64,
|
|
@@ -33962,21 +33962,29 @@ var init_CSVParser = __esm({
|
|
|
33962
33962
|
});
|
|
33963
33963
|
|
|
33964
33964
|
// ../../packages/parsing/src/DatasetCodec.ts
|
|
33965
|
-
var NULL_FIELD, NEEDS_ESCAPE, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
|
|
33965
|
+
var NULL_FIELD, EMPTY_FIELD, NEEDS_ESCAPE, QUOTE_CODE, QUOTE_TESTS, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
|
|
33966
33966
|
var init_DatasetCodec = __esm({
|
|
33967
33967
|
"../../packages/parsing/src/DatasetCodec.ts"() {
|
|
33968
33968
|
init_src();
|
|
33969
33969
|
NULL_FIELD = "";
|
|
33970
|
+
EMPTY_FIELD = '""';
|
|
33970
33971
|
NEEDS_ESCAPE = /["\n\r]/;
|
|
33972
|
+
QUOTE_CODE = 34;
|
|
33973
|
+
QUOTE_TESTS = /* @__PURE__ */ new Map([[",", /^\s|\s$|[",\n\r]/]]);
|
|
33971
33974
|
DatasetCodecClass = class {
|
|
33972
33975
|
constructor() {
|
|
33973
33976
|
/**
|
|
33974
|
-
* One value as an internal positional field
|
|
33975
|
-
*
|
|
33977
|
+
* One value as an internal positional field. Bare where it can be, quoted where it must be -
|
|
33978
|
+
* around the delimiter, a quote, a line break or an edge space - or the unquoted null marker
|
|
33979
|
+
* when there is no value. The delimiter is a parameter rather than a comma because `redelimit`
|
|
33980
|
+
* re-encodes a staged line with the export's own: a tab is safe bare in the staged file and is
|
|
33981
|
+
* not once that file is re-emitted tab-delimited.
|
|
33976
33982
|
*/
|
|
33977
|
-
this.encodeField = (value) => {
|
|
33983
|
+
this.encodeField = (value, delimiter = ",") => {
|
|
33978
33984
|
if (value === null || value === void 0) return NULL_FIELD;
|
|
33979
33985
|
const text = typeof value === "string" ? value : String(value);
|
|
33986
|
+
if (text === "") return EMPTY_FIELD;
|
|
33987
|
+
if (!this._quoteTest(delimiter).test(text)) return text;
|
|
33980
33988
|
if (!NEEDS_ESCAPE.test(text)) return `"${text}"`;
|
|
33981
33989
|
return `"${text.replaceAll('"', '""').replaceAll("\r", "\\r").replaceAll("\n", "\\n")}"`;
|
|
33982
33990
|
};
|
|
@@ -33988,7 +33996,7 @@ var init_DatasetCodec = __esm({
|
|
|
33988
33996
|
for (let index = 0; index < values.length; index++) {
|
|
33989
33997
|
if (index > 0)
|
|
33990
33998
|
row += delimiter;
|
|
33991
|
-
row += this.encodeField(values[index]);
|
|
33999
|
+
row += this.encodeField(values[index], delimiter);
|
|
33992
34000
|
}
|
|
33993
34001
|
return row;
|
|
33994
34002
|
};
|
|
@@ -34008,26 +34016,26 @@ var init_DatasetCodec = __esm({
|
|
|
34008
34016
|
let index = 0;
|
|
34009
34017
|
while (index < length) {
|
|
34010
34018
|
const char = line.charCodeAt(index);
|
|
34011
|
-
if (char ===
|
|
34019
|
+
if (char === QUOTE_CODE) {
|
|
34012
34020
|
if (!inQuotes) {
|
|
34013
34021
|
inQuotes = true;
|
|
34014
34022
|
hasQuotes = true;
|
|
34015
34023
|
fieldStart = index + 1;
|
|
34016
|
-
} else if (line.charCodeAt(index + 1) ===
|
|
34024
|
+
} else if (line.charCodeAt(index + 1) === QUOTE_CODE) {
|
|
34017
34025
|
index++;
|
|
34018
34026
|
} else {
|
|
34019
34027
|
inQuotes = false;
|
|
34020
34028
|
fieldEnd = index;
|
|
34021
34029
|
}
|
|
34022
34030
|
} else if (char === delimiterCode && !inQuotes) {
|
|
34023
|
-
fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
|
|
34031
|
+
fields.push(this._field(line, fieldStart, hasQuotes ? fieldEnd : index, hasQuotes));
|
|
34024
34032
|
fieldStart = index + 1;
|
|
34025
34033
|
fieldEnd = 0;
|
|
34026
34034
|
hasQuotes = false;
|
|
34027
34035
|
}
|
|
34028
34036
|
index++;
|
|
34029
34037
|
}
|
|
34030
|
-
fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
|
|
34038
|
+
fields.push(this._field(line, fieldStart, hasQuotes ? fieldEnd : length, hasQuotes));
|
|
34031
34039
|
return fields;
|
|
34032
34040
|
};
|
|
34033
34041
|
/**
|
|
@@ -34038,9 +34046,24 @@ var init_DatasetCodec = __esm({
|
|
|
34038
34046
|
Affirm_default(delimiter, "Invalid delimiter");
|
|
34039
34047
|
return this.encodeRow(this.decodeRow(line), delimiter);
|
|
34040
34048
|
};
|
|
34041
|
-
/**
|
|
34049
|
+
/**
|
|
34050
|
+
* The quoting test for one delimiter, built on first use. The four characters escaped into the
|
|
34051
|
+
* class are the ones a character class reads as syntax; anything else stands for itself there.
|
|
34052
|
+
*/
|
|
34053
|
+
this._quoteTest = (delimiter) => {
|
|
34054
|
+
let test = QUOTE_TESTS.get(delimiter);
|
|
34055
|
+
if (!test) {
|
|
34056
|
+
test = new RegExp(`^\\s|\\s$|["\\n\\r${delimiter.replace(/[\\\]^-]/g, "\\$&")}]`);
|
|
34057
|
+
QUOTE_TESTS.set(delimiter, test);
|
|
34058
|
+
}
|
|
34059
|
+
return test;
|
|
34060
|
+
};
|
|
34061
|
+
/**
|
|
34062
|
+
* One field's text, or null for the unquoted empty marker. A bare token carries its value as it
|
|
34063
|
+
* stands - nothing bare can hold a quote, so there is nothing to unescape in it.
|
|
34064
|
+
*/
|
|
34042
34065
|
this._field = (line, start, end, hasQuotes) => {
|
|
34043
|
-
if (!hasQuotes) return null;
|
|
34066
|
+
if (!hasQuotes) return start === end ? null : line.slice(start, end);
|
|
34044
34067
|
const raw = line.slice(start, end);
|
|
34045
34068
|
return raw.indexOf('"') === -1 ? raw : raw.replaceAll('""', '"');
|
|
34046
34069
|
};
|
|
@@ -34492,7 +34515,7 @@ var init_FixedWidthWriter = __esm({
|
|
|
34492
34515
|
* `lf` output needs nothing here and a `crlf` one needs the `\r` that turns that newline into a
|
|
34493
34516
|
* CRLF — which is exactly what a CRLF file looks like to anything that splits on `\n`.
|
|
34494
34517
|
*
|
|
34495
|
-
* A real positional feed picks one terminator and holds it:
|
|
34518
|
+
* A real positional feed picks one terminator and holds it: some are CRLF on
|
|
34496
34519
|
* every line including the last, and `lf` is the default everywhere else.
|
|
34497
34520
|
*/
|
|
34498
34521
|
this.lineSuffix = (output) => {
|
|
@@ -56695,7 +56718,7 @@ var init_OutputExecutor = __esm({
|
|
|
56695
56718
|
for (let index = 0; index < visibleKeys.length; index++) {
|
|
56696
56719
|
if (index > 0)
|
|
56697
56720
|
row += delimiter;
|
|
56698
|
-
row += DatasetCodec_default.encodeField(record[visibleKeys[index]]);
|
|
56721
|
+
row += DatasetCodec_default.encodeField(record[visibleKeys[index]], delimiter);
|
|
56699
56722
|
}
|
|
56700
56723
|
return row;
|
|
56701
56724
|
};
|
|
@@ -56880,9 +56903,10 @@ var init_OutputExecutor = __esm({
|
|
|
56880
56903
|
return resultList[0];
|
|
56881
56904
|
};
|
|
56882
56905
|
/**
|
|
56883
|
-
* The header line of a positional export: the visible field names, in column order,
|
|
56884
|
-
*
|
|
56885
|
-
* does. Null for a JSON/API output, where a
|
|
56906
|
+
* The header line of a positional export: the visible field names, in column order, written
|
|
56907
|
+
* through the SAME codec as the rows under them — so a name is quoted on the same rule a value
|
|
56908
|
+
* is, and the header parses with the reader the data does. Null for a JSON/API output, where a
|
|
56909
|
+
* record already carries its own field names, for a
|
|
56886
56910
|
* FIXED one, where a name does not fit the column it describes and the receiver reads by
|
|
56887
56911
|
* position anyway, and for an output that turns the header off.
|
|
56888
56912
|
*
|
|
@@ -56891,7 +56915,7 @@ var init_OutputExecutor = __esm({
|
|
|
56891
56915
|
this._composeHeader = (output, dsSchema) => {
|
|
56892
56916
|
if (output.header === false) return null;
|
|
56893
56917
|
if (output.format !== "CSV" && output.format !== "TXT") return null;
|
|
56894
|
-
return
|
|
56918
|
+
return DatasetCodec_default.encodeRow(DatasetSchema_default.names(dsSchema), this._getDelimiter(output));
|
|
56895
56919
|
};
|
|
56896
56920
|
/** The delimiter the exported file is written with. Only TXT configures it; CSV is a comma. */
|
|
56897
56921
|
this._getDelimiter = (output) => {
|
|
@@ -396,7 +396,7 @@
|
|
|
396
396
|
"lineEnding": {
|
|
397
397
|
"type": "string",
|
|
398
398
|
"enum": ["lf", "crlf"],
|
|
399
|
-
"description": "'FIXED' only. What ends every record, the last one included. Defaults to 'lf'. A real positional feed picks one and holds it —
|
|
399
|
+
"description": "'FIXED' only. What ends every record, the last one included. Defaults to 'lf'. A real positional feed picks one and holds it — some are CRLF on every line. A file with no terminator at all is fixed record framing (RECFM=FB) and is not expressible here."
|
|
400
400
|
},
|
|
401
401
|
"layouts": {
|
|
402
402
|
"type": "object",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.6.1",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"typescript"
|
|
45
45
|
],
|
|
46
46
|
"author": "",
|
|
47
|
-
"license": "
|
|
47
|
+
"license": "SEE LICENSE IN LICENCE.md",
|
|
48
48
|
"dependencies": {
|
|
49
49
|
"@aws-sdk/client-redshift-data": "^3.699.0",
|
|
50
50
|
"@aws-sdk/rds-signer": "^3.1136.0",
|