@t4h.framework/fs 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +23 -3
- package/.ai/skills/framework-fs-split/SKILL.md +350 -0
- package/README.md +297 -3
- package/dist/FileSystemClaim.d.ts +4 -0
- package/dist/FileSystemClaim.d.ts.map +1 -1
- package/dist/FileSystemClaim.js.map +1 -1
- package/dist/activities/FileJoinActivity.d.ts +59 -0
- package/dist/activities/FileJoinActivity.d.ts.map +1 -0
- package/dist/activities/FileJoinActivity.js +74 -0
- package/dist/activities/FileJoinActivity.js.map +1 -0
- package/dist/activities/FileReadActivity.d.ts +22 -0
- package/dist/activities/FileReadActivity.d.ts.map +1 -0
- package/dist/activities/FileReadActivity.js +68 -0
- package/dist/activities/FileReadActivity.js.map +1 -0
- package/dist/activities/FileSplitActivity.d.ts +31 -28
- package/dist/activities/FileSplitActivity.d.ts.map +1 -1
- package/dist/activities/FileSplitActivity.js +40 -34
- package/dist/activities/FileSplitActivity.js.map +1 -1
- package/dist/activities/__tests__/FileJoinActivity.spec.d.ts +2 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.d.ts.map +1 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.js +323 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.js.map +1 -0
- package/dist/activities/__tests__/FileReadActivity.spec.d.ts +2 -0
- package/dist/activities/__tests__/FileReadActivity.spec.d.ts.map +1 -0
- package/dist/activities/__tests__/FileReadActivity.spec.js +120 -0
- package/dist/activities/__tests__/FileReadActivity.spec.js.map +1 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.js +135 -132
- package/dist/activities/__tests__/FileSplitActivity.spec.js.map +1 -1
- package/dist/constants/operations.d.ts +3 -0
- package/dist/constants/operations.d.ts.map +1 -0
- package/dist/constants/operations.js +18 -0
- package/dist/constants/operations.js.map +1 -0
- package/dist/fs.d.ts +4 -0
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +4 -0
- package/dist/fs.js.map +1 -1
- package/dist/helpers/CSV.d.ts +14 -0
- package/dist/helpers/CSV.d.ts.map +1 -0
- package/dist/helpers/CSV.js +44 -0
- package/dist/helpers/CSV.js.map +1 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts +2 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts.map +1 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +196 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +1 -0
- package/dist/helpers/byte-delimiter-scanner.d.ts +43 -0
- package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -0
- package/dist/helpers/byte-delimiter-scanner.js +85 -0
- package/dist/helpers/byte-delimiter-scanner.js.map +1 -0
- package/dist/tools/assert-where-serializable.d.ts +2 -0
- package/dist/tools/assert-where-serializable.d.ts.map +1 -0
- package/dist/tools/assert-where-serializable.js +19 -0
- package/dist/tools/assert-where-serializable.js.map +1 -0
- package/package.json +11 -5
- package/dist/ActivityFileSystemClaim.d.ts +0 -7
- package/dist/ActivityFileSystemClaim.d.ts.map +0 -1
- package/dist/ActivityFileSystemClaim.js +0 -3
- package/dist/ActivityFileSystemClaim.js.map +0 -1
- package/dist/helpers/__tests__/csv-reader.spec.d.ts +0 -2
- package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +0 -1
- package/dist/helpers/__tests__/csv-reader.spec.js +0 -60
- package/dist/helpers/__tests__/csv-reader.spec.js.map +0 -1
- package/dist/helpers/csv-reader.d.ts +0 -8
- package/dist/helpers/csv-reader.d.ts.map +0 -1
- package/dist/helpers/csv-reader.js +0 -14
- package/dist/helpers/csv-reader.js.map +0 -1
- package/dist/helpers/parsers.d.ts +0 -9
- package/dist/helpers/parsers.d.ts.map +0 -1
- package/dist/helpers/parsers.js +0 -14
- package/dist/helpers/parsers.js.map +0 -1
- package/dist/testing/index.d.ts +0 -3
- package/dist/testing/index.d.ts.map +0 -1
- package/dist/testing/index.js +0 -32
- package/dist/testing/index.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# @t4h.framework/fs
|
|
2
2
|
|
|
3
|
-
File system claim for reading and writing files within activities
|
|
3
|
+
File system claim for reading and writing files within activities, plus the
|
|
4
|
+
workflow-facing file activities: `split` fans a large file out into one child
|
|
5
|
+
workflow per slice, `join` folds the batch results back into a single file,
|
|
6
|
+
and `read` brings a small byte window (a header, a trailer) into the workflow
|
|
7
|
+
as a string.
|
|
4
8
|
|
|
5
9
|
## Claim
|
|
6
10
|
|
|
@@ -16,8 +20,17 @@ import { FileSystemClaim } from '@t4h.framework/fs'
|
|
|
16
20
|
|
|
17
21
|
```typescript
|
|
18
22
|
abstract class FileSystemClaim {
|
|
19
|
-
public abstract write(
|
|
23
|
+
public abstract write(
|
|
24
|
+
path: string,
|
|
25
|
+
readable: Readable | Buffer,
|
|
26
|
+
): Binary | Promise<Binary>
|
|
20
27
|
public abstract read(path: string): Binary | Promise<Binary>
|
|
28
|
+
|
|
29
|
+
public abstract slice(
|
|
30
|
+
binary: Binary,
|
|
31
|
+
range: { start: number; end: number },
|
|
32
|
+
encoding?: BufferEncoding,
|
|
33
|
+
): Binary | Promise<Binary>
|
|
21
34
|
}
|
|
22
35
|
```
|
|
23
36
|
|
|
@@ -27,7 +40,11 @@ abstract class FileSystemClaim {
|
|
|
27
40
|
import { SyncActivity, Claim } from '@t4h.framework/core'
|
|
28
41
|
import { FileSystemClaim } from '@t4h.framework/fs'
|
|
29
42
|
|
|
30
|
-
class SaveReportActivity extends SyncActivity<
|
|
43
|
+
class SaveReportActivity extends SyncActivity<
|
|
44
|
+
[data: Buffer],
|
|
45
|
+
{ path: string },
|
|
46
|
+
any
|
|
47
|
+
> {
|
|
31
48
|
private readonly claims = new Claim({
|
|
32
49
|
fs: FileSystemClaim,
|
|
33
50
|
})
|
|
@@ -54,3 +71,280 @@ The runtime must supply a concrete implementation:
|
|
|
54
71
|
```typescript
|
|
55
72
|
{ provide: FileSystemClaim, value: new LocalFileSystemClaimImpl() }
|
|
56
73
|
```
|
|
74
|
+
|
|
75
|
+
## File read
|
|
76
|
+
|
|
77
|
+
### read
|
|
78
|
+
|
|
79
|
+
Reads a byte window of a `Binary` into the workflow as a decoded string. The
|
|
80
|
+
output is recorded in the workflow history (replay never re-reads the file),
|
|
81
|
+
which is why reads are capped at 1 MiB: this is the primitive for headers,
|
|
82
|
+
trailers and single records — a data file flows through `split`, never through
|
|
83
|
+
`read`.
|
|
84
|
+
|
|
85
|
+
```typescript
|
|
86
|
+
import { read } from '@t4h.framework/fs'
|
|
87
|
+
|
|
88
|
+
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
89
|
+
|
|
90
|
+
const trailer = await read(file, {
|
|
91
|
+
range: { start: file.contentLength - 240, end: file.contentLength },
|
|
92
|
+
})
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Options: `range` (default: the whole binary, subject to the same cap) and
|
|
96
|
+
`encoding` (default `'utf8'`).
|
|
97
|
+
|
|
98
|
+
The activity tries `FileSystemClaim.slice` first — on a file-backed binary a
|
|
99
|
+
trailer read is a ranged download, whatever the file size. Binaries the claim
|
|
100
|
+
cannot slice (e.g. an SFTP reference) fall back to streaming: the stream is
|
|
101
|
+
consumed up to the window and destroyed as soon as it is filled, so a header
|
|
102
|
+
read pulls the first chunks only, whatever the file size. The one expensive
|
|
103
|
+
case is a trailing window over a non-sliceable binary — the stream must
|
|
104
|
+
traverse the whole file to reach it.
|
|
105
|
+
|
|
106
|
+
Reading a slice is relative to the slice: `read(slice, { range: { start: 0,
|
|
107
|
+
end: 10 } })` returns the first 10 bytes of the slice, not of the underlying
|
|
108
|
+
file — slice ranges compose.
|
|
109
|
+
|
|
110
|
+
## File split
|
|
111
|
+
|
|
112
|
+
### split
|
|
113
|
+
|
|
114
|
+
Splits a file into byte-range slices, starts one child workflow per slice as
|
|
115
|
+
a single batch, and suspends the parent until every child is terminal —
|
|
116
|
+
regardless of each child succeeding or failing (a rejected child never
|
|
117
|
+
rejects the batch):
|
|
118
|
+
|
|
119
|
+
```typescript
|
|
120
|
+
import { split, Split } from '@t4h.framework/fs'
|
|
121
|
+
|
|
122
|
+
const processRecord = new Workflow(
|
|
123
|
+
{ id: 'process-record', schema: Type.String() },
|
|
124
|
+
async (record: string) => {
|
|
125
|
+
// record is the raw slice content
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
const main = new Workflow(fileSchema, async (file: Binary) => {
|
|
130
|
+
const batch = await split(file, processRecord, {
|
|
131
|
+
delimiter: Split.line(),
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
// this line only runs once the whole batch is terminal;
|
|
135
|
+
// batch: WorkflowBatchRef — { id: string, count: number }
|
|
136
|
+
})
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The design goal is constant memory regardless of file size:
|
|
140
|
+
|
|
141
|
+
1. The source stream is read **exactly once**, worker side. The same pass
|
|
142
|
+
persists a durable copy through `FileSystemClaim.write` while a byte
|
|
143
|
+
scanner records the slice offsets — no slice content is ever materialized
|
|
144
|
+
in the activity.
|
|
145
|
+
2. Children receive **range views** over the durable copy
|
|
146
|
+
(`FileSystemClaim.slice`).
|
|
147
|
+
3. The activity output is just the batch reference (`{ id, count }`), so the
|
|
148
|
+
parent history stays O(1) whatever the slice count. Per-child results are
|
|
149
|
+
never aggregated: they live with each child run, to be inspected
|
|
150
|
+
individually through the batch id.
|
|
151
|
+
|
|
152
|
+
Child workflows declare their real input type (e.g. `Type.String()`); how the
|
|
153
|
+
bytes reach them is a runtime concern.
|
|
154
|
+
|
|
155
|
+
### Delimiters
|
|
156
|
+
|
|
157
|
+
The cut is defined by the dev through a serializable delimiter spec. When the
|
|
158
|
+
delimiter is found at `[p, p + length)`, the previous slice ends at
|
|
159
|
+
`p + keep.previous` and the next slice starts at `p + length - keep.next`; the
|
|
160
|
+
bytes in between are discarded. The defaults discard the whole delimiter
|
|
161
|
+
(classic line semantics).
|
|
162
|
+
|
|
163
|
+
```typescript
|
|
164
|
+
Split.line() // cut at every '\n', delimiter discarded
|
|
165
|
+
Split.char(';') // cut at any byte sequence
|
|
166
|
+
Split.char(';', { previous: 1 }) // keep the ';' with the previous slice
|
|
167
|
+
Split.char('\n10', { next: 2 }) // positional files, see below
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The last form targets positional files where a logical record spans multiple
|
|
171
|
+
physical lines and a new record only starts when a line begins with a marker
|
|
172
|
+
(the record type code):
|
|
173
|
+
|
|
174
|
+
```
|
|
175
|
+
10JOHN;123\n20detail-a\n20detail-b\n10MARY;456\n20detail-c
|
|
176
|
+
└──────────── record 1 ───────────┘└─────── record 2 ──────┘
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
With `Split.char('\n10', { next: 2 })` the newline between records is
|
|
180
|
+
discarded, and the `10` marker — which is data, not separator — stays at the
|
|
181
|
+
start of the next slice.
|
|
182
|
+
|
|
183
|
+
### Options
|
|
184
|
+
|
|
185
|
+
```typescript
|
|
186
|
+
await split(file, workflow, {
|
|
187
|
+
delimiter: Split.line(), // required: where to cut
|
|
188
|
+
every: 10_000, // cut only every N occurrences (groups records)
|
|
189
|
+
trim: { start: 241, end: 241 }, // bytes cut off each end before scanning
|
|
190
|
+
skipEmpty: false, // keep zero-length slices (default: skipped)
|
|
191
|
+
encoding: 'latin1', // delimiter bytes + slice hydration (default: utf8)
|
|
192
|
+
timeoutInMs: 3_600_000, // max ms to await the batch; rejects on expiry
|
|
193
|
+
})
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
- **`every`** groups N records per child. With one million lines, `every:
|
|
197
|
+
10_000` starts 100 children instead of one million; each child receives a
|
|
198
|
+
raw block with its inner delimiters untouched and iterates locally. `keep`
|
|
199
|
+
arithmetic only applies at actual cut points.
|
|
200
|
+
- **`trim`** cuts a fixed number of bytes off each end of the file before
|
|
201
|
+
scanning — a positional header/trailer, which may not be aligned with the
|
|
202
|
+
record delimiter (a record-start delimiter like `Split.char('\n10')` fuses
|
|
203
|
+
the trailer into the last record's slice, so counting records would drop a
|
|
204
|
+
valid one along with it). Trimmed content never becomes a slice nor counts
|
|
205
|
+
toward `every`; fetch it with `read`. Getting the byte counts right is the
|
|
206
|
+
caller's business — positional layouts make them constants:
|
|
207
|
+
|
|
208
|
+
```typescript
|
|
209
|
+
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
210
|
+
|
|
211
|
+
const batch = await split(file, processInvoice, {
|
|
212
|
+
delimiter: Split.char('\n10', { next: 2 }),
|
|
213
|
+
trim: { start: 241, end: 241 }, // header + '\n', trailer + '\n'
|
|
214
|
+
})
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
- **`skipEmpty`** (default `true`) drops zero-length slices — consecutive
|
|
218
|
+
delimiters, or a file starting/ending with one — so they never become child
|
|
219
|
+
workflows. Turn it off when the slice index must align 1:1 with the record
|
|
220
|
+
position in the source file. "Empty" means zero bytes: a whitespace-only
|
|
221
|
+
line is a regular slice.
|
|
222
|
+
- **`encoding`** converts the delimiter to bytes and tells the runtime how to
|
|
223
|
+
hydrate each slice for the child. Offsets are always computed over raw
|
|
224
|
+
bytes, so multi-byte encodings never skew the ranges.
|
|
225
|
+
- **`timeoutInMs`** is the maximum time in milliseconds to await the batch. On
|
|
226
|
+
expiry the activity rejects (reason `timeout`) instead of suspending
|
|
227
|
+
forever — the safety net for a child that never reaches a terminal state.
|
|
228
|
+
|
|
229
|
+
## File join
|
|
230
|
+
|
|
231
|
+
The inverse of `split`: folds the results of a completed batch into a single
|
|
232
|
+
file. Outputs stream in one at a time through `WorkflowBatchClaim` and flow
|
|
233
|
+
straight into `FileSystemClaim.write` with backpressure, so memory stays
|
|
234
|
+
bounded whatever the batch size — only the resulting binary reaches the
|
|
235
|
+
workflow history.
|
|
236
|
+
|
|
237
|
+
```typescript
|
|
238
|
+
import { join, split, Split } from '@t4h.framework/fs'
|
|
239
|
+
|
|
240
|
+
const batch = await split(file, processRecord, { delimiter: Split.line() })
|
|
241
|
+
|
|
242
|
+
const csv = await join(batch, {
|
|
243
|
+
format: 'csv',
|
|
244
|
+
columns: [
|
|
245
|
+
{ key: 'id', header: 'Código' },
|
|
246
|
+
{ key: 'value', header: 'Valor (R$)' },
|
|
247
|
+
'date', // shorthand: key === header
|
|
248
|
+
],
|
|
249
|
+
})
|
|
250
|
+
// csv: Binary
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
### Formats
|
|
254
|
+
|
|
255
|
+
- **`lines`** — concatenates raw child outputs; the child owns the line
|
|
256
|
+
format. Options: `header`, `separator` (default `'\n'`).
|
|
257
|
+
- **`entries`** — NDJSON dump of `{ index, status, output }`, one entry per
|
|
258
|
+
line. The format-agnostic view of the batch, rejected children included by
|
|
259
|
+
default.
|
|
260
|
+
- **`csv`** — projects object outputs into RFC 4180 escaped columns. Options:
|
|
261
|
+
`columns` (required; each entry is `{ key, header? }` where `key` selects
|
|
262
|
+
the value on the child output — a dot-path like `'meta.total'` reaches
|
|
263
|
+
nested fields, matching the `where` semantics; a literal top-level key that
|
|
264
|
+
contains dots wins over the nested path — and `header` is the label on the
|
|
265
|
+
header row — a plain string is shorthand for `{ key }`), `delimiter`
|
|
266
|
+
(default `','`), `header` (default `true`; emits the column labels as the
|
|
267
|
+
first row).
|
|
268
|
+
|
|
269
|
+
### Status filter
|
|
270
|
+
|
|
271
|
+
`status` selects which child results make it into the file, instead of an
|
|
272
|
+
error policy: it defaults to `['fulfilled']` for `lines` and `csv`, and to
|
|
273
|
+
both statuses for `entries`. Filtering is silent by design — a file of
|
|
274
|
+
rejected children for reprocessing is one call away:
|
|
275
|
+
|
|
276
|
+
```typescript
|
|
277
|
+
const failures = await join(batch, { format: 'entries', status: ['rejected'] })
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
### Where filter
|
|
281
|
+
|
|
282
|
+
`where` narrows results with a [sift](https://github.com/crcn/sift.js)
|
|
283
|
+
(MongoDB-style) query over each entry's output, applied after the `status`
|
|
284
|
+
filter. The accepted subset is deliberate — only declarative, JSON-only
|
|
285
|
+
operators: `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`, `$nin`,
|
|
286
|
+
`$exists`, `$not`, `$and`, `$or`, `$nor`, plus plain equality and dot-paths
|
|
287
|
+
for nested fields:
|
|
288
|
+
|
|
289
|
+
```typescript
|
|
290
|
+
const relevant = await join(batch, {
|
|
291
|
+
format: 'csv',
|
|
292
|
+
columns: ['id', 'total'],
|
|
293
|
+
where: {
|
|
294
|
+
valid: true, // plain strict equality
|
|
295
|
+
total: { $gt: 100 }, // operators
|
|
296
|
+
'meta.situacao': 'aprovada', // dot-path into nested output
|
|
297
|
+
$or: [{ ref: 'a' }, { ref: 'b' }],
|
|
298
|
+
},
|
|
299
|
+
})
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
`join` also accepts the child output type as a generic — top-level keys then
|
|
303
|
+
narrow to its fields, with values typed by the field (dot-paths are accepted
|
|
304
|
+
without narrowing):
|
|
305
|
+
|
|
306
|
+
```typescript
|
|
307
|
+
type InvoiceResult = { id: string; valid: boolean; total: number }
|
|
308
|
+
|
|
309
|
+
await join<InvoiceResult>(batch, {
|
|
310
|
+
format: 'csv',
|
|
311
|
+
columns: ['id'],
|
|
312
|
+
where: { valid: true, total: { $gt: 100 } }, // { valid: 'yes' } rejects
|
|
313
|
+
})
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
Anything that cannot serialize losslessly into the workflow history is
|
|
317
|
+
rejected eagerly at `toInput` with a `TypeError`: functions (including
|
|
318
|
+
`$where`), `RegExp` values (including `$regex`), `undefined`, and operators
|
|
319
|
+
outside the subset — a dropped closure would make the replay filter
|
|
320
|
+
differently than the first execution. For arbitrary predicates, write a
|
|
321
|
+
custom activity in project code (the function then lives in versioned code,
|
|
322
|
+
not in history).
|
|
323
|
+
|
|
324
|
+
Two MongoDB semantics worth knowing: array fields match by containment
|
|
325
|
+
(`{ tags: 'a' }` matches `tags: ['a', 'b']`), and equality against an object
|
|
326
|
+
value compares deeply (never by reference). Comparisons never coerce types.
|
|
327
|
+
|
|
328
|
+
Filtering runs inside the existing streaming loop, over outputs that are
|
|
329
|
+
already hydrated one at a time — the query compiles once and adds one
|
|
330
|
+
predicate call per entry, no extra serialization and no change to the memory
|
|
331
|
+
profile. It is applied app-side: every child output is still fetched and
|
|
332
|
+
parsed; `where` decides what enters the file, not what leaves the database.
|
|
333
|
+
|
|
334
|
+
### ByteDelimiterScanner
|
|
335
|
+
|
|
336
|
+
The scanner behind `split` is exported for direct use. It is a pass-through
|
|
337
|
+
`Transform`: chunks flow through unchanged (so the same pass can persist the
|
|
338
|
+
file) while it records slice offsets in absolute byte positions, handling
|
|
339
|
+
delimiters split across chunk boundaries. `ranges` is complete once the stream
|
|
340
|
+
has flushed.
|
|
341
|
+
|
|
342
|
+
```typescript
|
|
343
|
+
import { ByteDelimiterScanner } from '@t4h.framework/fs'
|
|
344
|
+
|
|
345
|
+
const scanner = new ByteDelimiterScanner({ delimiter: { value: '\n' } })
|
|
346
|
+
|
|
347
|
+
await pipeline(source, scanner, destination)
|
|
348
|
+
|
|
349
|
+
scanner.ranges // [{ start: 0, end: 3 }, ...]
|
|
350
|
+
```
|
|
@@ -3,5 +3,9 @@ import type { Binary } from '@t4h.framework/core';
|
|
|
3
3
|
export declare abstract class FileSystemClaim {
|
|
4
4
|
abstract write(path: string, readable: Readable | Buffer): Binary | Promise<Binary>;
|
|
5
5
|
abstract read(path: string): Binary | Promise<Binary>;
|
|
6
|
+
abstract slice(binary: Binary, range: {
|
|
7
|
+
start: number;
|
|
8
|
+
end: number;
|
|
9
|
+
}, encoding?: BufferEncoding): Binary | Promise<Binary>;
|
|
6
10
|
}
|
|
7
11
|
//# sourceMappingURL=FileSystemClaim.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemClaim.d.ts","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAEjD,8BAAsB,eAAe;aACnB,KAAK,CACnB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAEX,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"FileSystemClaim.d.ts","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAEjD,8BAAsB,eAAe;aACnB,KAAK,CACnB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAEX,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAE5C,KAAK,CACnB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EACrC,QAAQ,CAAC,EAAE,cAAc,GACxB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAC5B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemClaim.js","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,eAAe;
|
|
1
|
+
{"version":3,"file":"FileSystemClaim.js","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,eAAe;CAapC"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { type Binary, type Serializable, SyncActivity, type WorkflowBatchRef } from '@t4h.framework/core';
|
|
2
|
+
import { CSV } from '../helpers/CSV.js';
|
|
3
|
+
export type FileJoinStatus = 'fulfilled' | 'rejected';
|
|
4
|
+
type FileJoinWhereOperators<TValue> = {
|
|
5
|
+
$eq?: TValue;
|
|
6
|
+
$ne?: TValue;
|
|
7
|
+
$lt?: TValue;
|
|
8
|
+
$lte?: TValue;
|
|
9
|
+
$gt?: TValue;
|
|
10
|
+
$gte?: TValue;
|
|
11
|
+
$in?: TValue[];
|
|
12
|
+
$nin?: TValue[];
|
|
13
|
+
$exists?: boolean;
|
|
14
|
+
$not?: FileJoinWhereCondition<TValue>;
|
|
15
|
+
};
|
|
16
|
+
type FileJoinWhereCondition<TValue> = TValue | FileJoinWhereOperators<TValue>;
|
|
17
|
+
export type FileJoinWhere<TOutput = unknown> = ([TOutput] extends [object] ? {
|
|
18
|
+
[K in keyof TOutput]?: FileJoinWhereCondition<Exclude<TOutput[K], undefined>>;
|
|
19
|
+
} & {
|
|
20
|
+
[path: `${string}.${string}`]: Serializable;
|
|
21
|
+
} : Record<string, Serializable>) & {
|
|
22
|
+
$and?: FileJoinWhere<TOutput>[];
|
|
23
|
+
$or?: FileJoinWhere<TOutput>[];
|
|
24
|
+
$nor?: FileJoinWhere<TOutput>[];
|
|
25
|
+
};
|
|
26
|
+
type FileJoinOptionsBase<TOutput> = {
|
|
27
|
+
status?: FileJoinStatus[];
|
|
28
|
+
where?: FileJoinWhere<TOutput>;
|
|
29
|
+
};
|
|
30
|
+
type FileJoinOptionsLines<TOutput> = FileJoinOptionsBase<TOutput> & {
|
|
31
|
+
format: 'lines';
|
|
32
|
+
header?: string;
|
|
33
|
+
separator?: string;
|
|
34
|
+
};
|
|
35
|
+
type FileJoinOptionsEntries<TOutput> = FileJoinOptionsBase<TOutput> & {
|
|
36
|
+
format: 'entries';
|
|
37
|
+
};
|
|
38
|
+
type FileJoinOptionsCsv<TOutput> = FileJoinOptionsBase<TOutput> & {
|
|
39
|
+
format: 'csv';
|
|
40
|
+
columns: CSV.Column[];
|
|
41
|
+
delimiter?: string;
|
|
42
|
+
header?: boolean;
|
|
43
|
+
};
|
|
44
|
+
export type FileJoinOptions<TOutput = unknown> = FileJoinOptionsLines<TOutput> | FileJoinOptionsEntries<TOutput> | FileJoinOptionsCsv<TOutput>;
|
|
45
|
+
export type FileJoinActivityInput = {
|
|
46
|
+
batch: string;
|
|
47
|
+
options: FileJoinOptions;
|
|
48
|
+
};
|
|
49
|
+
export declare class FileJoinActivity extends SyncActivity<readonly [batch: WorkflowBatchRef, options: FileJoinOptions], FileJoinActivityInput, Binary, Binary> {
|
|
50
|
+
private readonly claims;
|
|
51
|
+
toInput(batch: WorkflowBatchRef, options: FileJoinOptions): FileJoinActivityInput;
|
|
52
|
+
run({ batch, options }: FileJoinActivityInput): Promise<Binary>;
|
|
53
|
+
toOutput(output: Binary): Binary;
|
|
54
|
+
private writeHeader;
|
|
55
|
+
private toLine;
|
|
56
|
+
}
|
|
57
|
+
export declare function join<TOutput = unknown>(batch: WorkflowBatchRef, options: FileJoinOptions<TOutput>): Promise<Binary>;
|
|
58
|
+
export {};
|
|
59
|
+
//# sourceMappingURL=FileJoinActivity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,KAAK,YAAY,EACjB,YAAY,EAGZ,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAGvC,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,CAAA;AAErD,KAAK,sBAAsB,CAAC,MAAM,IAAI;IACpC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,EAAE,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,IAAI,CAAC,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAA;CACtC,CAAA;AAED,KAAK,sBAAsB,CAAC,MAAM,IAAI,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAA;AAE7E,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,GACtE;KACG,CAAC,IAAI,MAAM,OAAO,CAAC,CAAC,EAAE,sBAAsB,CAC3C,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAC/B;CACF,GAAG;IACF,CAAC,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,GAAG,YAAY,CAAA;CAC5C,GACD,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,GAAG;IAClC,IAAI,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;IAC/B,GAAG,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;IAC9B,IAAI,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;CAChC,CAAA;AAED,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAQD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,eAAe,CAAC,EAC5D,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAMX,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAsCrE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,WAAW;IAazB,OAAO,CAAC,MAAM;CAef;AAED,wBAAsB,IAAI,CAAC,OAAO,GAAG,OAAO,EAC1C,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC,CAMjB"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { once } from 'node:events';
|
|
2
|
+
import { PassThrough } from 'node:stream';
|
|
3
|
+
import { Claim, History, SyncActivity, WorkflowBatchClaim, } from '@t4h.framework/core';
|
|
4
|
+
import sift from 'sift';
|
|
5
|
+
import { OPERATIONS } from '../constants/operations.js';
|
|
6
|
+
import { FileSystemClaim } from '../FileSystemClaim.js';
|
|
7
|
+
import { CSV } from '../helpers/CSV.js';
|
|
8
|
+
import { assertWhereSerializable } from '../tools/assert-where-serializable.js';
|
|
9
|
+
const extensions = {
|
|
10
|
+
lines: 'txt',
|
|
11
|
+
entries: 'ndjson',
|
|
12
|
+
csv: 'csv',
|
|
13
|
+
};
|
|
14
|
+
export class FileJoinActivity extends SyncActivity {
|
|
15
|
+
claims = new Claim({
|
|
16
|
+
fs: FileSystemClaim,
|
|
17
|
+
batch: WorkflowBatchClaim,
|
|
18
|
+
});
|
|
19
|
+
toInput(batch, options) {
|
|
20
|
+
if (options.where)
|
|
21
|
+
assertWhereSerializable(options.where, 'where');
|
|
22
|
+
return { batch: batch.id, options };
|
|
23
|
+
}
|
|
24
|
+
async run({ batch, options }) {
|
|
25
|
+
const stream = new PassThrough();
|
|
26
|
+
const writing = this.claims.fs.write(`join/${this.id}.${extensions[options.format]}`, stream);
|
|
27
|
+
const write = async (line) => {
|
|
28
|
+
if (!stream.write(line))
|
|
29
|
+
await once(stream, 'drain');
|
|
30
|
+
};
|
|
31
|
+
const statuses = new Set(options.status ?? ['fulfilled']);
|
|
32
|
+
if (!options.status && options.format === 'entries')
|
|
33
|
+
statuses.add('rejected');
|
|
34
|
+
const matches = options.where &&
|
|
35
|
+
sift.createQueryTester(options.where, {
|
|
36
|
+
operations: OPERATIONS,
|
|
37
|
+
});
|
|
38
|
+
await this.writeHeader(options, write);
|
|
39
|
+
for await (const entry of this.claims.batch.outputs(batch)) {
|
|
40
|
+
if (!statuses.has(entry.status))
|
|
41
|
+
continue;
|
|
42
|
+
if (matches && !matches(entry.output))
|
|
43
|
+
continue;
|
|
44
|
+
await write(this.toLine(entry, options));
|
|
45
|
+
}
|
|
46
|
+
stream.end();
|
|
47
|
+
return await writing;
|
|
48
|
+
}
|
|
49
|
+
toOutput(output) {
|
|
50
|
+
return output;
|
|
51
|
+
}
|
|
52
|
+
async writeHeader(options, write) {
|
|
53
|
+
if (options.format === 'lines' && options.header)
|
|
54
|
+
await write(options.header + (options.separator ?? '\n'));
|
|
55
|
+
if (options.format === 'csv' && options.header !== false)
|
|
56
|
+
await write(CSV.toHeader(options.columns, { delimiter: options.delimiter }));
|
|
57
|
+
}
|
|
58
|
+
toLine(entry, options) {
|
|
59
|
+
if (options.format === 'entries')
|
|
60
|
+
return JSON.stringify(entry) + '\n';
|
|
61
|
+
if (options.format === 'csv')
|
|
62
|
+
return CSV.toLine(entry.output, options.columns, {
|
|
63
|
+
delimiter: options.delimiter,
|
|
64
|
+
});
|
|
65
|
+
const value = typeof entry.output === 'string'
|
|
66
|
+
? entry.output
|
|
67
|
+
: JSON.stringify(entry.output);
|
|
68
|
+
return value + (options.separator ?? '\n');
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export async function join(batch, options) {
|
|
72
|
+
return await History.reconciler(FileJoinActivity, batch, options);
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=FileJoinActivity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,EAEP,YAAY,EACZ,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAA;AAiE/E,MAAM,UAAU,GAA8C;IAC5D,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,QAAQ;IACjB,GAAG,EAAE,KAAK;CACX,CAAA;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,KAAuB,EACvB,OAAwB;QAExB,IAAI,OAAO,CAAC,KAAK;YAAE,uBAAuB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAElE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;IACrC,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAyB;QACxD,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAA;QAEhC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAClC,QAAQ,IAAI,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC/C,MAAM,CACP,CAAA;QAED,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,CAAC,CAAA;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAiB,OAAO,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YACjD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAE1B,MAAM,OAAO,GACX,OAAO,CAAC,KAAK;YACb,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,EAAE;gBACpC,UAAU,EAAE,UAAU;aACvB,CAAC,CAAA;QAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAEtC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAEzC,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAE/C,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;QAC1C,CAAC;QAED,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAwB,EACxB,KAAsC;QAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM;YAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAA;QAE3D,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YACtD,MAAM,KAAK,CACT,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAChE,CAAA;IACL,CAAC;IAEO,MAAM,CAAC,KAA0B,EAAE,OAAwB;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;QAErE,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YAC1B,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAA;QAEJ,MAAM,KAAK,GACT,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;YAC9B,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAElC,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAA;IAC5C,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,KAAuB,EACvB,OAAiC;IAEjC,OAAO,MAAM,OAAO,CAAC,UAAU,CAC7B,gBAAgB,EAChB,KAAK,EACL,OAA0B,CAC3B,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type Binary, SyncActivity } from '@t4h.framework/core';
|
|
2
|
+
export type FileReadOptions = {
|
|
3
|
+
range?: {
|
|
4
|
+
start: number;
|
|
5
|
+
end: number;
|
|
6
|
+
};
|
|
7
|
+
encoding?: BufferEncoding;
|
|
8
|
+
};
|
|
9
|
+
export type FileReadActivityInput = {
|
|
10
|
+
binary: Binary;
|
|
11
|
+
options: FileReadOptions;
|
|
12
|
+
};
|
|
13
|
+
export declare class FileReadActivity extends SyncActivity<readonly [binary: Binary, options: FileReadOptions], FileReadActivityInput, string, string> {
|
|
14
|
+
private readonly claims;
|
|
15
|
+
toInput(binary: Binary, options: FileReadOptions): FileReadActivityInput;
|
|
16
|
+
run({ binary, options, }: FileReadActivityInput): Promise<string>;
|
|
17
|
+
toOutput(output: string): string;
|
|
18
|
+
private slice;
|
|
19
|
+
private window;
|
|
20
|
+
}
|
|
21
|
+
export declare function read(binary: Binary, options?: FileReadOptions): Promise<string>;
|
|
22
|
+
//# sourceMappingURL=FileReadActivity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAID,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;CAuCrB;AAED,wBAAsB,IAAI,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAEjB"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { buffer } from 'node:stream/consumers';
|
|
2
|
+
import { Claim, History, SyncActivity } from '@t4h.framework/core';
|
|
3
|
+
import { FileSystemClaim } from '../FileSystemClaim.js';
|
|
4
|
+
const FILE_READ_MAX_BYTES = 1024 * 1024;
|
|
5
|
+
export class FileReadActivity extends SyncActivity {
|
|
6
|
+
claims = new Claim({
|
|
7
|
+
fs: FileSystemClaim,
|
|
8
|
+
});
|
|
9
|
+
toInput(binary, options) {
|
|
10
|
+
return { binary, options };
|
|
11
|
+
}
|
|
12
|
+
async run({ binary, options, }) {
|
|
13
|
+
const encoding = options.encoding ?? 'utf8';
|
|
14
|
+
const range = options.range ?? { start: 0, end: binary.contentLength };
|
|
15
|
+
const length = range.end - range.start;
|
|
16
|
+
if (!Number.isInteger(range.start) || range.start < 0 || length < 0)
|
|
17
|
+
throw new TypeError('Read range must be a non-negative byte window');
|
|
18
|
+
if (range.end > binary.contentLength)
|
|
19
|
+
throw new TypeError('Read range must not exceed the binary length');
|
|
20
|
+
if (length > FILE_READ_MAX_BYTES)
|
|
21
|
+
throw new RangeError(`Length ${length} exceeded max ${FILE_READ_MAX_BYTES}`);
|
|
22
|
+
const slice = await this.slice(binary, range, encoding);
|
|
23
|
+
if (slice)
|
|
24
|
+
return (await buffer(await slice.stream())).toString(encoding);
|
|
25
|
+
return (await this.window(await binary.stream(), range)).toString(encoding);
|
|
26
|
+
}
|
|
27
|
+
toOutput(output) {
|
|
28
|
+
return output;
|
|
29
|
+
}
|
|
30
|
+
async slice(binary, range, encoding) {
|
|
31
|
+
try {
|
|
32
|
+
return await this.claims.fs.slice(binary, range, encoding);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async window(readable, range) {
|
|
39
|
+
const chunks = [];
|
|
40
|
+
let position = 0;
|
|
41
|
+
let size = 0;
|
|
42
|
+
try {
|
|
43
|
+
for await (const data of readable) {
|
|
44
|
+
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
45
|
+
const chunkStart = position;
|
|
46
|
+
position += chunk.length;
|
|
47
|
+
if (position <= range.start)
|
|
48
|
+
continue;
|
|
49
|
+
const from = Math.max(range.start - chunkStart, 0);
|
|
50
|
+
const to = Math.min(range.end - chunkStart, chunk.length);
|
|
51
|
+
chunks.push(chunk.subarray(from, to));
|
|
52
|
+
size += to - from;
|
|
53
|
+
if (position >= range.end)
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
readable.destroy();
|
|
59
|
+
}
|
|
60
|
+
if (size < range.end - range.start)
|
|
61
|
+
throw new RangeError('Binary stream ended before the read window was filled');
|
|
62
|
+
return Buffer.concat(chunks);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export async function read(binary, options = {}) {
|
|
66
|
+
return await History.reconciler(FileReadActivity, binary, options);
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=FileReadActivity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileReadActivity.js","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EAAe,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAE/E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAYvD,MAAM,mBAAmB,GAAG,IAAI,GAAG,IAAI,CAAA;AAEvC,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;KACpB,CAAC,CAAA;IAEK,OAAO,CACZ,MAAc,EACd,OAAwB;QAExB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACe;QACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAA;QAE3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,aAAa,EAAE,CAAA;QAEtE,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAA;QAEtC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC;YACjE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAA;QAEtE,IAAI,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,aAAa;YAClC,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAA;QAErE,IAAI,MAAM,GAAG,mBAAmB;YAC9B,MAAM,IAAI,UAAU,CAClB,UAAU,MAAM,iBAAiB,mBAAmB,EAAE,CACvD,CAAA;QAEH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;QAEvD,IAAI,KAAK;YAAE,OAAO,CAAC,MAAM,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEzE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAC7E,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,MAAc,EACd,KAAqC,EACrC,QAAwB;QAExB,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAClB,QAAkB,EAClB,KAAqC;QAErC,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,IAAI,IAAI,GAAG,CAAC,CAAA;QAEZ,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAClC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAE9D,MAAM,UAAU,GAAG,QAAQ,CAAA;gBAE3B,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;gBAExB,IAAI,QAAQ,IAAI,KAAK,CAAC,KAAK;oBAAE,SAAQ;gBAErC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,EAAE,CAAC,CAAC,CAAA;gBAClD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;gBAEzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;gBAErC,IAAI,IAAI,EAAE,GAAG,IAAI,CAAA;gBAEjB,IAAI,QAAQ,IAAI,KAAK,CAAC,GAAG;oBAAE,MAAK;YAClC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,QAAQ,CAAC,OAAO,EAAE,CAAA;QACpB,CAAC;QAED,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK;YAChC,MAAM,IAAI,UAAU,CAClB,uDAAuD,CACxD,CAAA;QAEH,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,UAA2B,EAAE;IAE7B,OAAO,MAAM,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;AACpE,CAAC"}
|
|
@@ -1,36 +1,39 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import { AsyncActivity, type Binary, type Workflow, WorkflowBatchRef, workflowBatchSchema } from '@t4h.framework/core';
|
|
2
|
+
import { type ByteTrim, type FileSplitDelimiter } from '../helpers/byte-delimiter-scanner.js';
|
|
3
|
+
export type FileSplitOptions = {
|
|
4
|
+
delimiter: FileSplitDelimiter;
|
|
5
|
+
every?: number;
|
|
6
|
+
trim?: ByteTrim;
|
|
7
|
+
skipEmpty?: boolean;
|
|
6
8
|
encoding?: BufferEncoding;
|
|
9
|
+
timeoutInMs?: number;
|
|
7
10
|
};
|
|
8
|
-
type FileSplitOptionsLine = FileSplitOptionsBase<'txt' | 'ndjson'> & {
|
|
9
|
-
trimEnd?: boolean;
|
|
10
|
-
};
|
|
11
|
-
type FileSplitOptionsCSV = FileSplitOptionsBase<'csv'> & {
|
|
12
|
-
delimiter?: string;
|
|
13
|
-
};
|
|
14
|
-
type FileSplitOptionsCustom = FileSplitOptionsBase<string> & {
|
|
15
|
-
delimiter?: string;
|
|
16
|
-
trimEnd?: boolean;
|
|
17
|
-
};
|
|
18
|
-
type FileSplitOptions = FileSplitOptionsLine | FileSplitOptionsCSV | FileSplitOptionsCustom;
|
|
19
11
|
export type FileSplitActivityInput = {
|
|
20
12
|
binary: Binary;
|
|
21
|
-
|
|
13
|
+
workflow: string;
|
|
14
|
+
options: FileSplitOptions;
|
|
22
15
|
};
|
|
23
|
-
export
|
|
16
|
+
export type FileSplitBatch = {
|
|
17
|
+
batch: string;
|
|
18
|
+
count: number;
|
|
19
|
+
};
|
|
20
|
+
export declare class FileSplitActivity extends AsyncActivity<readonly [
|
|
21
|
+
binary: Binary,
|
|
22
|
+
workflow: Workflow<any, any>,
|
|
23
|
+
options: FileSplitOptions
|
|
24
|
+
], FileSplitActivityInput, FileSplitBatch, WorkflowBatchRef, FileSplitActivityInput, typeof workflowBatchSchema> {
|
|
24
25
|
private readonly claims;
|
|
25
|
-
toInput(binary: Binary, options
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
toInput(binary: Binary, workflow: Workflow<any, any>, options: FileSplitOptions): FileSplitActivityInput;
|
|
27
|
+
bootstrap({ binary, workflow, options, }: FileSplitActivityInput): Promise<import("@t4h.framework/core").AsyncActivityResultPending<FileSplitActivityInput, import("typebox").TObject<{
|
|
28
|
+
batch: import("typebox").TString;
|
|
29
|
+
count: import("typebox").TNumber;
|
|
30
|
+
}>> | import("@t4h.framework/core").AsyncActivityResultFulfilled<FileSplitBatch | undefined>>;
|
|
31
|
+
advance(payload: FileSplitBatch): import("@t4h.framework/core").AsyncActivityResultFulfilled<FileSplitBatch | undefined>;
|
|
32
|
+
toOutput(output: FileSplitBatch): WorkflowBatchRef;
|
|
33
|
+
}
|
|
34
|
+
export declare function split(binary: Binary, workflow: Workflow<any, any>, options: FileSplitOptions): Promise<WorkflowBatchRef>;
|
|
35
|
+
export declare namespace Split {
|
|
36
|
+
const line: (value?: string) => FileSplitDelimiter;
|
|
37
|
+
const char: (value: string, keep?: FileSplitDelimiter["keep"]) => FileSplitDelimiter;
|
|
28
38
|
}
|
|
29
|
-
type BinaryWorkflow = Workflow<Type.TObject<{
|
|
30
|
-
binary: Type.TSchema;
|
|
31
|
-
}>, any>;
|
|
32
|
-
export declare function split(binary: Binary, workflow: BinaryWorkflow, options?: FileSplitOptions & {
|
|
33
|
-
parallel?: boolean;
|
|
34
|
-
}): Promise<AsyncActivityAwaitable<StartWorkflowActivity>[]>;
|
|
35
|
-
export {};
|
|
36
39
|
//# sourceMappingURL=FileSplitActivity.d.ts.map
|