@t4h.framework/fs 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/.ai/skills/framework-fs/SKILL.md +23 -3
  2. package/.ai/skills/framework-fs-split/SKILL.md +350 -0
  3. package/README.md +297 -3
  4. package/dist/FileSystemClaim.d.ts +4 -0
  5. package/dist/FileSystemClaim.d.ts.map +1 -1
  6. package/dist/FileSystemClaim.js.map +1 -1
  7. package/dist/activities/FileJoinActivity.d.ts +59 -0
  8. package/dist/activities/FileJoinActivity.d.ts.map +1 -0
  9. package/dist/activities/FileJoinActivity.js +74 -0
  10. package/dist/activities/FileJoinActivity.js.map +1 -0
  11. package/dist/activities/FileReadActivity.d.ts +22 -0
  12. package/dist/activities/FileReadActivity.d.ts.map +1 -0
  13. package/dist/activities/FileReadActivity.js +68 -0
  14. package/dist/activities/FileReadActivity.js.map +1 -0
  15. package/dist/activities/FileSplitActivity.d.ts +31 -28
  16. package/dist/activities/FileSplitActivity.d.ts.map +1 -1
  17. package/dist/activities/FileSplitActivity.js +40 -34
  18. package/dist/activities/FileSplitActivity.js.map +1 -1
  19. package/dist/activities/__tests__/FileJoinActivity.spec.d.ts +2 -0
  20. package/dist/activities/__tests__/FileJoinActivity.spec.d.ts.map +1 -0
  21. package/dist/activities/__tests__/FileJoinActivity.spec.js +323 -0
  22. package/dist/activities/__tests__/FileJoinActivity.spec.js.map +1 -0
  23. package/dist/activities/__tests__/FileReadActivity.spec.d.ts +2 -0
  24. package/dist/activities/__tests__/FileReadActivity.spec.d.ts.map +1 -0
  25. package/dist/activities/__tests__/FileReadActivity.spec.js +120 -0
  26. package/dist/activities/__tests__/FileReadActivity.spec.js.map +1 -0
  27. package/dist/activities/__tests__/FileSplitActivity.spec.js +135 -132
  28. package/dist/activities/__tests__/FileSplitActivity.spec.js.map +1 -1
  29. package/dist/constants/operations.d.ts +3 -0
  30. package/dist/constants/operations.d.ts.map +1 -0
  31. package/dist/constants/operations.js +18 -0
  32. package/dist/constants/operations.js.map +1 -0
  33. package/dist/fs.d.ts +4 -0
  34. package/dist/fs.d.ts.map +1 -1
  35. package/dist/fs.js +4 -0
  36. package/dist/fs.js.map +1 -1
  37. package/dist/helpers/CSV.d.ts +14 -0
  38. package/dist/helpers/CSV.d.ts.map +1 -0
  39. package/dist/helpers/CSV.js +44 -0
  40. package/dist/helpers/CSV.js.map +1 -0
  41. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts +2 -0
  42. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts.map +1 -0
  43. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +196 -0
  44. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +1 -0
  45. package/dist/helpers/byte-delimiter-scanner.d.ts +43 -0
  46. package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -0
  47. package/dist/helpers/byte-delimiter-scanner.js +85 -0
  48. package/dist/helpers/byte-delimiter-scanner.js.map +1 -0
  49. package/dist/tools/assert-where-serializable.d.ts +2 -0
  50. package/dist/tools/assert-where-serializable.d.ts.map +1 -0
  51. package/dist/tools/assert-where-serializable.js +19 -0
  52. package/dist/tools/assert-where-serializable.js.map +1 -0
  53. package/package.json +11 -5
  54. package/dist/ActivityFileSystemClaim.d.ts +0 -7
  55. package/dist/ActivityFileSystemClaim.d.ts.map +0 -1
  56. package/dist/ActivityFileSystemClaim.js +0 -3
  57. package/dist/ActivityFileSystemClaim.js.map +0 -1
  58. package/dist/helpers/__tests__/csv-reader.spec.d.ts +0 -2
  59. package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +0 -1
  60. package/dist/helpers/__tests__/csv-reader.spec.js +0 -60
  61. package/dist/helpers/__tests__/csv-reader.spec.js.map +0 -1
  62. package/dist/helpers/csv-reader.d.ts +0 -8
  63. package/dist/helpers/csv-reader.d.ts.map +0 -1
  64. package/dist/helpers/csv-reader.js +0 -14
  65. package/dist/helpers/csv-reader.js.map +0 -1
  66. package/dist/helpers/parsers.d.ts +0 -9
  67. package/dist/helpers/parsers.d.ts.map +0 -1
  68. package/dist/helpers/parsers.js +0 -14
  69. package/dist/helpers/parsers.js.map +0 -1
  70. package/dist/testing/index.d.ts +0 -3
  71. package/dist/testing/index.d.ts.map +0 -1
  72. package/dist/testing/index.js +0 -32
  73. package/dist/testing/index.js.map +0 -1
package/README.md CHANGED
@@ -1,6 +1,10 @@
1
1
  # @t4h.framework/fs
2
2
 
3
- File system claim for reading and writing files within activities.
3
+ File system claim for reading and writing files within activities, plus the
4
+ workflow-facing file activities: `split` fans a large file out into one child
5
+ workflow per slice, `join` folds the batch results back into a single file,
6
+ and `read` brings a small byte window (a header, a trailer) into the workflow
7
+ as a string.
4
8
 
5
9
  ## Claim
6
10
 
@@ -16,8 +20,17 @@ import { FileSystemClaim } from '@t4h.framework/fs'
16
20
 
17
21
  ```typescript
18
22
  abstract class FileSystemClaim {
19
- public abstract write(path: string, readable: Readable | Buffer): Binary | Promise<Binary>
23
+ public abstract write(
24
+ path: string,
25
+ readable: Readable | Buffer,
26
+ ): Binary | Promise<Binary>
20
27
  public abstract read(path: string): Binary | Promise<Binary>
28
+
29
+ public abstract slice(
30
+ binary: Binary,
31
+ range: { start: number; end: number },
32
+ encoding?: BufferEncoding,
33
+ ): Binary | Promise<Binary>
21
34
  }
22
35
  ```
23
36
 
@@ -27,7 +40,11 @@ abstract class FileSystemClaim {
27
40
  import { SyncActivity, Claim } from '@t4h.framework/core'
28
41
  import { FileSystemClaim } from '@t4h.framework/fs'
29
42
 
30
- class SaveReportActivity extends SyncActivity<[data: Buffer], { path: string }, any> {
43
+ class SaveReportActivity extends SyncActivity<
44
+ [data: Buffer],
45
+ { path: string },
46
+ any
47
+ > {
31
48
  private readonly claims = new Claim({
32
49
  fs: FileSystemClaim,
33
50
  })
@@ -54,3 +71,280 @@ The runtime must supply a concrete implementation:
54
71
  ```typescript
55
72
  { provide: FileSystemClaim, value: new LocalFileSystemClaimImpl() }
56
73
  ```
74
+
75
+ ## File read
76
+
77
+ ### read
78
+
79
+ Reads a byte window of a `Binary` into the workflow as a decoded string. The
80
+ output is recorded in the workflow history (replay never re-reads the file),
81
+ which is why reads are capped at 1 MiB: this is the primitive for headers,
82
+ trailers and single records — a data file flows through `split`, never through
83
+ `read`.
84
+
85
+ ```typescript
86
+ import { read } from '@t4h.framework/fs'
87
+
88
+ const header = await read(file, { range: { start: 0, end: 240 } })
89
+
90
+ const trailer = await read(file, {
91
+ range: { start: file.contentLength - 240, end: file.contentLength },
92
+ })
93
+ ```
94
+
95
+ Options: `range` (default: the whole binary, subject to the same cap) and
96
+ `encoding` (default `'utf8'`).
97
+
98
+ The activity tries `FileSystemClaim.slice` first — on a file-backed binary a
99
+ trailer read is a ranged download, whatever the file size. Binaries the claim
100
+ cannot slice (e.g. an SFTP reference) fall back to streaming: the stream is
101
+ consumed up to the window and destroyed as soon as it is filled, so a header
102
+ read pulls the first chunks only, whatever the file size. The one expensive
103
+ case is a trailing window over a non-sliceable binary — the stream must
104
+ traverse the whole file to reach it.
105
+
106
+ Reading a slice is relative to the slice: `read(slice, { range: { start: 0,
107
+ end: 10 } })` returns the first 10 bytes of the slice, not of the underlying
108
+ file — slice ranges compose.
109
+
110
+ ## File split
111
+
112
+ ### split
113
+
114
+ Splits a file into byte-range slices, starts one child workflow per slice as
115
+ a single batch, and suspends the parent until every child is terminal —
116
+ regardless of each child succeeding or failing (a rejected child never
117
+ rejects the batch):
118
+
119
+ ```typescript
120
+ import { split, Split } from '@t4h.framework/fs'
121
+
122
+ const processRecord = new Workflow(
123
+ { id: 'process-record', schema: Type.String() },
124
+ async (record: string) => {
125
+ // record is the raw slice content
126
+ },
127
+ )
128
+
129
+ const main = new Workflow(fileSchema, async (file: Binary) => {
130
+ const batch = await split(file, processRecord, {
131
+ delimiter: Split.line(),
132
+ })
133
+
134
+ // this line only runs once the whole batch is terminal;
135
+ // batch: WorkflowBatchRef — { id: string, count: number }
136
+ })
137
+ ```
138
+
139
+ The design goal is constant memory regardless of file size:
140
+
141
+ 1. The source stream is read **exactly once**, worker side. The same pass
142
+ persists a durable copy through `FileSystemClaim.write` while a byte
143
+ scanner records the slice offsets — no slice content is ever materialized
144
+ in the activity.
145
+ 2. Children receive **range views** over the durable copy
146
+ (`FileSystemClaim.slice`).
147
+ 3. The activity output is just the batch reference (`{ id, count }`), so the
148
+ parent history stays O(1) whatever the slice count. Per-child results are
149
+ never aggregated: they live with each child run, to be inspected
150
+ individually through the batch id.
151
+
152
+ Child workflows declare their real input type (e.g. `Type.String()`); how the
153
+ bytes reach them is a runtime concern.
154
+
155
+ ### Delimiters
156
+
157
+ The cut is defined by the dev through a serializable delimiter spec. When the
158
+ delimiter is found at `[p, p + length)`, the previous slice ends at
159
+ `p + keep.previous` and the next slice starts at `p + length - keep.next`; the
160
+ bytes in between are discarded. The defaults discard the whole delimiter
161
+ (classic line semantics).
162
+
163
+ ```typescript
164
+ Split.line() // cut at every '\n', delimiter discarded
165
+ Split.char(';') // cut at any byte sequence
166
+ Split.char(';', { previous: 1 }) // keep the ';' with the previous slice
167
+ Split.char('\n10', { next: 2 }) // positional files, see below
168
+ ```
169
+
170
+ The last form targets positional files where a logical record spans multiple
171
+ physical lines and a new record only starts when a line begins with a marker
172
+ (the record type code):
173
+
174
+ ```
175
+ 10JOHN;123\n20detail-a\n20detail-b\n10MARY;456\n20detail-c
176
+ └──────────── record 1 ───────────┘└─────── record 2 ──────┘
177
+ ```
178
+
179
+ With `Split.char('\n10', { next: 2 })` the newline between records is
180
+ discarded, and the `10` marker — which is data, not separator — stays at the
181
+ start of the next slice.
182
+
183
+ ### Options
184
+
185
+ ```typescript
186
+ await split(file, workflow, {
187
+ delimiter: Split.line(), // required: where to cut
188
+ every: 10_000, // cut only every N occurrences (groups records)
189
+ trim: { start: 241, end: 241 }, // bytes cut off each end before scanning
190
+ skipEmpty: false, // keep zero-length slices (default: skipped)
191
+ encoding: 'latin1', // delimiter bytes + slice hydration (default: utf8)
192
+ timeoutInMs: 3_600_000, // max ms to await the batch; rejects on expiry
193
+ })
194
+ ```
195
+
196
+ - **`every`** groups N records per child. With one million lines, `every:
197
+ 10_000` starts 100 children instead of one million; each child receives a
198
+ raw block with its inner delimiters untouched and iterates locally. `keep`
199
+ arithmetic only applies at actual cut points.
200
+ - **`trim`** cuts a fixed number of bytes off each end of the file before
201
+ scanning — a positional header/trailer, which may not be aligned with the
202
+ record delimiter (a record-start delimiter like `Split.char('\n10')` fuses
203
+ the trailer into the last record's slice, so counting records would drop a
204
+ valid one along with it). Trimmed content never becomes a slice nor counts
205
+ toward `every`; fetch it with `read`. Getting the byte counts right is the
206
+ caller's business — positional layouts make them constants:
207
+
208
+ ```typescript
209
+ const header = await read(file, { range: { start: 0, end: 240 } })
210
+
211
+ const batch = await split(file, processInvoice, {
212
+ delimiter: Split.char('\n10', { next: 2 }),
213
+ trim: { start: 241, end: 241 }, // header + '\n', trailer + '\n'
214
+ })
215
+ ```
216
+
217
+ - **`skipEmpty`** (default `true`) drops zero-length slices — consecutive
218
+ delimiters, or a file starting/ending with one — so they never become child
219
+ workflows. Turn it off when the slice index must align 1:1 with the record
220
+ position in the source file. "Empty" means zero bytes: a whitespace-only
221
+ line is a regular slice.
222
+ - **`encoding`** converts the delimiter to bytes and tells the runtime how to
223
+ hydrate each slice for the child. Offsets are always computed over raw
224
+ bytes, so multi-byte encodings never skew the ranges.
225
+ - **`timeoutInMs`** is the maximum time in milliseconds to await the batch. On
226
+ expiry the activity rejects (reason `timeout`) instead of suspending
227
+ forever — the safety net for a child that never reaches a terminal state.
228
+
229
+ ## File join
230
+
231
+ The inverse of `split`: folds the results of a completed batch into a single
232
+ file. Outputs stream in one at a time through `WorkflowBatchClaim` and flow
233
+ straight into `FileSystemClaim.write` with backpressure, so memory stays
234
+ bounded whatever the batch size — only the resulting binary reaches the
235
+ workflow history.
236
+
237
+ ```typescript
238
+ import { join, split, Split } from '@t4h.framework/fs'
239
+
240
+ const batch = await split(file, processRecord, { delimiter: Split.line() })
241
+
242
+ const csv = await join(batch, {
243
+ format: 'csv',
244
+ columns: [
245
+ { key: 'id', header: 'Código' },
246
+ { key: 'value', header: 'Valor (R$)' },
247
+ 'date', // shorthand: key === header
248
+ ],
249
+ })
250
+ // csv: Binary
251
+ ```
252
+
253
+ ### Formats
254
+
255
+ - **`lines`** — concatenates raw child outputs; the child owns the line
256
+ format. Options: `header`, `separator` (default `'\n'`).
257
+ - **`entries`** — NDJSON dump of `{ index, status, output }`, one entry per
258
+ line. The format-agnostic view of the batch, rejected children included by
259
+ default.
260
+ - **`csv`** — projects object outputs into RFC 4180 escaped columns. Options:
261
+ `columns` (required; each entry is `{ key, header? }` where `key` selects
262
+ the value on the child output — a dot-path like `'meta.total'` reaches
263
+ nested fields, matching the `where` semantics; a literal top-level key that
264
+ contains dots wins over the nested path — and `header` is the label on the
265
+ header row — a plain string is shorthand for `{ key }`), `delimiter`
266
+ (default `','`), `header` (default `true`; emits the column labels as the
267
+ first row).
268
+
269
+ ### Status filter
270
+
271
+ `status` selects which child results make it into the file, instead of an
272
+ error policy: it defaults to `['fulfilled']` for `lines` and `csv`, and to
273
+ both statuses for `entries`. Filtering is silent by design — a file of
274
+ rejected children for reprocessing is one call away:
275
+
276
+ ```typescript
277
+ const failures = await join(batch, { format: 'entries', status: ['rejected'] })
278
+ ```
279
+
280
+ ### Where filter
281
+
282
+ `where` narrows results with a [sift](https://github.com/crcn/sift.js)
283
+ (MongoDB-style) query over each entry's output, applied after the `status`
284
+ filter. The accepted subset is deliberate — only declarative, JSON-only
285
+ operators: `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`, `$nin`,
286
+ `$exists`, `$not`, `$and`, `$or`, `$nor`, plus plain equality and dot-paths
287
+ for nested fields:
288
+
289
+ ```typescript
290
+ const relevant = await join(batch, {
291
+ format: 'csv',
292
+ columns: ['id', 'total'],
293
+ where: {
294
+ valid: true, // plain strict equality
295
+ total: { $gt: 100 }, // operators
296
+ 'meta.situacao': 'aprovada', // dot-path into nested output
297
+ $or: [{ ref: 'a' }, { ref: 'b' }],
298
+ },
299
+ })
300
+ ```
301
+
302
+ `join` also accepts the child output type as a generic — top-level keys then
303
+ narrow to its fields, with values typed by the field (dot-paths are accepted
304
+ without narrowing):
305
+
306
+ ```typescript
307
+ type InvoiceResult = { id: string; valid: boolean; total: number }
308
+
309
+ await join<InvoiceResult>(batch, {
310
+ format: 'csv',
311
+ columns: ['id'],
312
+ where: { valid: true, total: { $gt: 100 } }, // { valid: 'yes' } rejects
313
+ })
314
+ ```
315
+
316
+ Anything that cannot serialize losslessly into the workflow history is
317
+ rejected eagerly at `toInput` with a `TypeError`: functions (including
318
+ `$where`), `RegExp` values (including `$regex`), `undefined`, and operators
319
+ outside the subset — a dropped closure would make the replay filter
320
+ differently than the first execution. For arbitrary predicates, write a
321
+ custom activity in project code (the function then lives in versioned code,
322
+ not in history).
323
+
324
+ Two MongoDB semantics worth knowing: array fields match by containment
325
+ (`{ tags: 'a' }` matches `tags: ['a', 'b']`), and equality against an object
326
+ value compares deeply (never by reference). Comparisons never coerce types.
327
+
328
+ Filtering runs inside the existing streaming loop, over outputs that are
329
+ already hydrated one at a time — the query compiles once and adds one
330
+ predicate call per entry, no extra serialization and no change to the memory
331
+ profile. It is applied app-side: every child output is still fetched and
332
+ parsed; `where` decides what enters the file, not what leaves the database.
333
+
334
+ ### ByteDelimiterScanner
335
+
336
+ The scanner behind `split` is exported for direct use. It is a pass-through
337
+ `Transform`: chunks flow through unchanged (so the same pass can persist the
338
+ file) while it records slice offsets in absolute byte positions, handling
339
+ delimiters split across chunk boundaries. `ranges` is complete once the stream
340
+ has flushed.
341
+
342
+ ```typescript
343
+ import { ByteDelimiterScanner } from '@t4h.framework/fs'
344
+
345
+ const scanner = new ByteDelimiterScanner({ delimiter: { value: '\n' } })
346
+
347
+ await pipeline(source, scanner, destination)
348
+
349
+ scanner.ranges // [{ start: 0, end: 3 }, ...]
350
+ ```
@@ -3,5 +3,9 @@ import type { Binary } from '@t4h.framework/core';
3
3
  export declare abstract class FileSystemClaim {
4
4
  abstract write(path: string, readable: Readable | Buffer): Binary | Promise<Binary>;
5
5
  abstract read(path: string): Binary | Promise<Binary>;
6
+ abstract slice(binary: Binary, range: {
7
+ start: number;
8
+ end: number;
9
+ }, encoding?: BufferEncoding): Binary | Promise<Binary>;
6
10
  }
7
11
  //# sourceMappingURL=FileSystemClaim.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"FileSystemClaim.d.ts","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAEjD,8BAAsB,eAAe;aACnB,KAAK,CACnB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAEX,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAC7D"}
1
+ {"version":3,"file":"FileSystemClaim.d.ts","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAEjD,8BAAsB,eAAe;aACnB,KAAK,CACnB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAEX,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAE5C,KAAK,CACnB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EACrC,QAAQ,CAAC,EAAE,cAAc,GACxB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAC5B"}
@@ -1 +1 @@
1
- {"version":3,"file":"FileSystemClaim.js","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,eAAe;CAOpC"}
1
+ {"version":3,"file":"FileSystemClaim.js","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,eAAe;CAapC"}
@@ -0,0 +1,59 @@
1
+ import { type Binary, type Serializable, SyncActivity, type WorkflowBatchRef } from '@t4h.framework/core';
2
+ import { CSV } from '../helpers/CSV.js';
3
+ export type FileJoinStatus = 'fulfilled' | 'rejected';
4
+ type FileJoinWhereOperators<TValue> = {
5
+ $eq?: TValue;
6
+ $ne?: TValue;
7
+ $lt?: TValue;
8
+ $lte?: TValue;
9
+ $gt?: TValue;
10
+ $gte?: TValue;
11
+ $in?: TValue[];
12
+ $nin?: TValue[];
13
+ $exists?: boolean;
14
+ $not?: FileJoinWhereCondition<TValue>;
15
+ };
16
+ type FileJoinWhereCondition<TValue> = TValue | FileJoinWhereOperators<TValue>;
17
+ export type FileJoinWhere<TOutput = unknown> = ([TOutput] extends [object] ? {
18
+ [K in keyof TOutput]?: FileJoinWhereCondition<Exclude<TOutput[K], undefined>>;
19
+ } & {
20
+ [path: `${string}.${string}`]: Serializable;
21
+ } : Record<string, Serializable>) & {
22
+ $and?: FileJoinWhere<TOutput>[];
23
+ $or?: FileJoinWhere<TOutput>[];
24
+ $nor?: FileJoinWhere<TOutput>[];
25
+ };
26
+ type FileJoinOptionsBase<TOutput> = {
27
+ status?: FileJoinStatus[];
28
+ where?: FileJoinWhere<TOutput>;
29
+ };
30
+ type FileJoinOptionsLines<TOutput> = FileJoinOptionsBase<TOutput> & {
31
+ format: 'lines';
32
+ header?: string;
33
+ separator?: string;
34
+ };
35
+ type FileJoinOptionsEntries<TOutput> = FileJoinOptionsBase<TOutput> & {
36
+ format: 'entries';
37
+ };
38
+ type FileJoinOptionsCsv<TOutput> = FileJoinOptionsBase<TOutput> & {
39
+ format: 'csv';
40
+ columns: CSV.Column[];
41
+ delimiter?: string;
42
+ header?: boolean;
43
+ };
44
+ export type FileJoinOptions<TOutput = unknown> = FileJoinOptionsLines<TOutput> | FileJoinOptionsEntries<TOutput> | FileJoinOptionsCsv<TOutput>;
45
+ export type FileJoinActivityInput = {
46
+ batch: string;
47
+ options: FileJoinOptions;
48
+ };
49
+ export declare class FileJoinActivity extends SyncActivity<readonly [batch: WorkflowBatchRef, options: FileJoinOptions], FileJoinActivityInput, Binary, Binary> {
50
+ private readonly claims;
51
+ toInput(batch: WorkflowBatchRef, options: FileJoinOptions): FileJoinActivityInput;
52
+ run({ batch, options }: FileJoinActivityInput): Promise<Binary>;
53
+ toOutput(output: Binary): Binary;
54
+ private writeHeader;
55
+ private toLine;
56
+ }
57
+ export declare function join<TOutput = unknown>(batch: WorkflowBatchRef, options: FileJoinOptions<TOutput>): Promise<Binary>;
58
+ export {};
59
+ //# sourceMappingURL=FileJoinActivity.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,KAAK,YAAY,EACjB,YAAY,EAGZ,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAGvC,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,CAAA;AAErD,KAAK,sBAAsB,CAAC,MAAM,IAAI;IACpC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,EAAE,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,IAAI,CAAC,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAA;CACtC,CAAA;AAED,KAAK,sBAAsB,CAAC,MAAM,IAAI,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAA;AAE7E,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,GACtE;KACG,CAAC,IAAI,MAAM,OAAO,CAAC,CAAC,EAAE,sBAAsB,CAC3C,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAC/B;CACF,GAAG;IACF,CAAC,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,GAAG,YAAY,CAAA;CAC5C,GACD,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,GAAG;IAClC,IAAI,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;IAC/B,GAAG,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;IAC9B,IAAI,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;CAChC,CAAA;AAED,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAQD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,eAAe,CAAC,EAC5D,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAMX,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAsCrE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,WAAW;IAazB,OAAO,CAAC,MAAM;CAef;AAED,wBAAsB,IAAI,CAAC,OAAO,GAAG,OAAO,EAC1C,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC,CAMjB"}
@@ -0,0 +1,74 @@
1
+ import { once } from 'node:events';
2
+ import { PassThrough } from 'node:stream';
3
+ import { Claim, History, SyncActivity, WorkflowBatchClaim, } from '@t4h.framework/core';
4
+ import sift from 'sift';
5
+ import { OPERATIONS } from '../constants/operations.js';
6
+ import { FileSystemClaim } from '../FileSystemClaim.js';
7
+ import { CSV } from '../helpers/CSV.js';
8
+ import { assertWhereSerializable } from '../tools/assert-where-serializable.js';
9
+ const extensions = {
10
+ lines: 'txt',
11
+ entries: 'ndjson',
12
+ csv: 'csv',
13
+ };
14
+ export class FileJoinActivity extends SyncActivity {
15
+ claims = new Claim({
16
+ fs: FileSystemClaim,
17
+ batch: WorkflowBatchClaim,
18
+ });
19
+ toInput(batch, options) {
20
+ if (options.where)
21
+ assertWhereSerializable(options.where, 'where');
22
+ return { batch: batch.id, options };
23
+ }
24
+ async run({ batch, options }) {
25
+ const stream = new PassThrough();
26
+ const writing = this.claims.fs.write(`join/${this.id}.${extensions[options.format]}`, stream);
27
+ const write = async (line) => {
28
+ if (!stream.write(line))
29
+ await once(stream, 'drain');
30
+ };
31
+ const statuses = new Set(options.status ?? ['fulfilled']);
32
+ if (!options.status && options.format === 'entries')
33
+ statuses.add('rejected');
34
+ const matches = options.where &&
35
+ sift.createQueryTester(options.where, {
36
+ operations: OPERATIONS,
37
+ });
38
+ await this.writeHeader(options, write);
39
+ for await (const entry of this.claims.batch.outputs(batch)) {
40
+ if (!statuses.has(entry.status))
41
+ continue;
42
+ if (matches && !matches(entry.output))
43
+ continue;
44
+ await write(this.toLine(entry, options));
45
+ }
46
+ stream.end();
47
+ return await writing;
48
+ }
49
+ toOutput(output) {
50
+ return output;
51
+ }
52
+ async writeHeader(options, write) {
53
+ if (options.format === 'lines' && options.header)
54
+ await write(options.header + (options.separator ?? '\n'));
55
+ if (options.format === 'csv' && options.header !== false)
56
+ await write(CSV.toHeader(options.columns, { delimiter: options.delimiter }));
57
+ }
58
+ toLine(entry, options) {
59
+ if (options.format === 'entries')
60
+ return JSON.stringify(entry) + '\n';
61
+ if (options.format === 'csv')
62
+ return CSV.toLine(entry.output, options.columns, {
63
+ delimiter: options.delimiter,
64
+ });
65
+ const value = typeof entry.output === 'string'
66
+ ? entry.output
67
+ : JSON.stringify(entry.output);
68
+ return value + (options.separator ?? '\n');
69
+ }
70
+ }
71
+ export async function join(batch, options) {
72
+ return await History.reconciler(FileJoinActivity, batch, options);
73
+ }
74
+ //# sourceMappingURL=FileJoinActivity.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,EAEP,YAAY,EACZ,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAA;AAiE/E,MAAM,UAAU,GAA8C;IAC5D,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,QAAQ;IACjB,GAAG,EAAE,KAAK;CACX,CAAA;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,KAAuB,EACvB,OAAwB;QAExB,IAAI,OAAO,CAAC,KAAK;YAAE,uBAAuB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAElE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;IACrC,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAyB;QACxD,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAA;QAEhC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAClC,QAAQ,IAAI,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC/C,MAAM,CACP,CAAA;QAED,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,CAAC,CAAA;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAiB,OAAO,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YACjD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAE1B,MAAM,OAAO,GACX,OAAO,CAAC,KAAK;YACb,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,EAAE;gBACpC,UAAU,EAAE,UAAU;aACvB,CAAC,CAAA;QAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAEtC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAEzC,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAE/C,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;QAC1C,CAAC;QAED,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAwB,EACxB,KAAsC;QAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM;YAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAA;QAE3D,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YACtD,MAAM,KAAK,CACT,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAChE,CAAA;IACL,CAAC;IAEO,MAAM,CAAC,KAA0B,EAAE,OAAwB;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;QAErE,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YAC1B,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAA;QAEJ,MAAM,KAAK,GACT,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;YAC9B,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAElC,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAA;IAC5C,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,KAAuB,EACvB,OAAiC;IAEjC,OAAO,MAAM,OAAO,CAAC,UAAU,CAC7B,gBAAgB,EAChB,KAAK,EACL,OAA0B,CAC3B,CAAA;AACH,CAAC"}
@@ -0,0 +1,22 @@
1
+ import { type Binary, SyncActivity } from '@t4h.framework/core';
2
+ export type FileReadOptions = {
3
+ range?: {
4
+ start: number;
5
+ end: number;
6
+ };
7
+ encoding?: BufferEncoding;
8
+ };
9
+ export type FileReadActivityInput = {
10
+ binary: Binary;
11
+ options: FileReadOptions;
12
+ };
13
+ export declare class FileReadActivity extends SyncActivity<readonly [binary: Binary, options: FileReadOptions], FileReadActivityInput, string, string> {
14
+ private readonly claims;
15
+ toInput(binary: Binary, options: FileReadOptions): FileReadActivityInput;
16
+ run({ binary, options, }: FileReadActivityInput): Promise<string>;
17
+ toOutput(output: string): string;
18
+ private slice;
19
+ private window;
20
+ }
21
+ export declare function read(binary: Binary, options?: FileReadOptions): Promise<string>;
22
+ //# sourceMappingURL=FileReadActivity.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAID,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;CAuCrB;AAED,wBAAsB,IAAI,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAEjB"}
@@ -0,0 +1,68 @@
1
+ import { buffer } from 'node:stream/consumers';
2
+ import { Claim, History, SyncActivity } from '@t4h.framework/core';
3
+ import { FileSystemClaim } from '../FileSystemClaim.js';
4
+ const FILE_READ_MAX_BYTES = 1024 * 1024;
5
+ export class FileReadActivity extends SyncActivity {
6
+ claims = new Claim({
7
+ fs: FileSystemClaim,
8
+ });
9
+ toInput(binary, options) {
10
+ return { binary, options };
11
+ }
12
+ async run({ binary, options, }) {
13
+ const encoding = options.encoding ?? 'utf8';
14
+ const range = options.range ?? { start: 0, end: binary.contentLength };
15
+ const length = range.end - range.start;
16
+ if (!Number.isInteger(range.start) || range.start < 0 || length < 0)
17
+ throw new TypeError('Read range must be a non-negative byte window');
18
+ if (range.end > binary.contentLength)
19
+ throw new TypeError('Read range must not exceed the binary length');
20
+ if (length > FILE_READ_MAX_BYTES)
21
+ throw new RangeError(`Length ${length} exceeded max ${FILE_READ_MAX_BYTES}`);
22
+ const slice = await this.slice(binary, range, encoding);
23
+ if (slice)
24
+ return (await buffer(await slice.stream())).toString(encoding);
25
+ return (await this.window(await binary.stream(), range)).toString(encoding);
26
+ }
27
+ toOutput(output) {
28
+ return output;
29
+ }
30
+ async slice(binary, range, encoding) {
31
+ try {
32
+ return await this.claims.fs.slice(binary, range, encoding);
33
+ }
34
+ catch {
35
+ return null;
36
+ }
37
+ }
38
+ async window(readable, range) {
39
+ const chunks = [];
40
+ let position = 0;
41
+ let size = 0;
42
+ try {
43
+ for await (const data of readable) {
44
+ const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data);
45
+ const chunkStart = position;
46
+ position += chunk.length;
47
+ if (position <= range.start)
48
+ continue;
49
+ const from = Math.max(range.start - chunkStart, 0);
50
+ const to = Math.min(range.end - chunkStart, chunk.length);
51
+ chunks.push(chunk.subarray(from, to));
52
+ size += to - from;
53
+ if (position >= range.end)
54
+ break;
55
+ }
56
+ }
57
+ finally {
58
+ readable.destroy();
59
+ }
60
+ if (size < range.end - range.start)
61
+ throw new RangeError('Binary stream ended before the read window was filled');
62
+ return Buffer.concat(chunks);
63
+ }
64
+ }
65
+ export async function read(binary, options = {}) {
66
+ return await History.reconciler(FileReadActivity, binary, options);
67
+ }
68
+ //# sourceMappingURL=FileReadActivity.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FileReadActivity.js","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EAAe,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAE/E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAYvD,MAAM,mBAAmB,GAAG,IAAI,GAAG,IAAI,CAAA;AAEvC,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;KACpB,CAAC,CAAA;IAEK,OAAO,CACZ,MAAc,EACd,OAAwB;QAExB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACe;QACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAA;QAE3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,aAAa,EAAE,CAAA;QAEtE,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAA;QAEtC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC;YACjE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAA;QAEtE,IAAI,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,aAAa;YAClC,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAA;QAErE,IAAI,MAAM,GAAG,mBAAmB;YAC9B,MAAM,IAAI,UAAU,CAClB,UAAU,MAAM,iBAAiB,mBAAmB,EAAE,CACvD,CAAA;QAEH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;QAEvD,IAAI,KAAK;YAAE,OAAO,CAAC,MAAM,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEzE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAC7E,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,MAAc,EACd,KAAqC,EACrC,QAAwB;QAExB,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAClB,QAAkB,EAClB,KAAqC;QAErC,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,IAAI,IAAI,GAAG,CAAC,CAAA;QAEZ,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAClC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAE9D,MAAM,UAAU,GAAG,QAAQ,CAAA;gBAE3B,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;gBAExB,IAAI,QAAQ,IAAI,KAAK,CAAC,KAAK;oBAAE,SAAQ;gBAErC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,EAAE,CAAC,CAAC,CAAA;gBAClD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;gBAEzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;gBAErC,IAAI,IAAI,EAAE,GAAG,IAAI,CAAA;gBAEjB,IAAI,QAAQ,IAAI,KAAK,CAAC,GAAG;oBAAE,MAAK;YAClC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,QAAQ,CAAC,OAAO,EAAE,CAAA;QACpB,CAAC;QAED,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK;YAChC,MAAM,IAAI,UAAU,CAClB,uDAAuD,CACxD,CAAA;QAEH,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,UAA2B,EAAE;IAE7B,OAAO,MAAM,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;AACpE,CAAC"}
@@ -1,36 +1,39 @@
1
- import { AsyncActivityAwaitable, type Binary, StartWorkflowActivity, SyncActivity, Workflow } from '@t4h.framework/core';
2
- import type Type from 'typebox';
3
- import { type ParserFormat } from '../helpers/parsers.js';
4
- export type FileSplitOptionsBase<TFormat extends ParserFormat> = {
5
- format: TFormat;
1
+ import { AsyncActivity, type Binary, type Workflow, WorkflowBatchRef, workflowBatchSchema } from '@t4h.framework/core';
2
+ import { type ByteTrim, type FileSplitDelimiter } from '../helpers/byte-delimiter-scanner.js';
3
+ export type FileSplitOptions = {
4
+ delimiter: FileSplitDelimiter;
5
+ every?: number;
6
+ trim?: ByteTrim;
7
+ skipEmpty?: boolean;
6
8
  encoding?: BufferEncoding;
9
+ timeoutInMs?: number;
7
10
  };
8
- type FileSplitOptionsLine = FileSplitOptionsBase<'txt' | 'ndjson'> & {
9
- trimEnd?: boolean;
10
- };
11
- type FileSplitOptionsCSV = FileSplitOptionsBase<'csv'> & {
12
- delimiter?: string;
13
- };
14
- type FileSplitOptionsCustom = FileSplitOptionsBase<string> & {
15
- delimiter?: string;
16
- trimEnd?: boolean;
17
- };
18
- type FileSplitOptions = FileSplitOptionsLine | FileSplitOptionsCSV | FileSplitOptionsCustom;
19
11
  export type FileSplitActivityInput = {
20
12
  binary: Binary;
21
- options?: FileSplitOptions;
13
+ workflow: string;
14
+ options: FileSplitOptions;
22
15
  };
23
- export declare class FileSplitActivity extends SyncActivity<readonly [binary: Binary, options?: FileSplitOptions], FileSplitActivityInput, Binary[], Binary[]> {
16
+ export type FileSplitBatch = {
17
+ batch: string;
18
+ count: number;
19
+ };
20
+ export declare class FileSplitActivity extends AsyncActivity<readonly [
21
+ binary: Binary,
22
+ workflow: Workflow<any, any>,
23
+ options: FileSplitOptions
24
+ ], FileSplitActivityInput, FileSplitBatch, WorkflowBatchRef, FileSplitActivityInput, typeof workflowBatchSchema> {
24
25
  private readonly claims;
25
- toInput(binary: Binary, options?: FileSplitOptions): FileSplitActivityInput;
26
- run({ binary, options, }: FileSplitActivityInput): Promise<Binary[]>;
27
- toOutput(binaries: Binary[]): Binary[];
26
+ toInput(binary: Binary, workflow: Workflow<any, any>, options: FileSplitOptions): FileSplitActivityInput;
27
+ bootstrap({ binary, workflow, options, }: FileSplitActivityInput): Promise<import("@t4h.framework/core").AsyncActivityResultPending<FileSplitActivityInput, import("typebox").TObject<{
28
+ batch: import("typebox").TString;
29
+ count: import("typebox").TNumber;
30
+ }>> | import("@t4h.framework/core").AsyncActivityResultFulfilled<FileSplitBatch | undefined>>;
31
+ advance(payload: FileSplitBatch): import("@t4h.framework/core").AsyncActivityResultFulfilled<FileSplitBatch | undefined>;
32
+ toOutput(output: FileSplitBatch): WorkflowBatchRef;
33
+ }
34
+ export declare function split(binary: Binary, workflow: Workflow<any, any>, options: FileSplitOptions): Promise<WorkflowBatchRef>;
35
+ export declare namespace Split {
36
+ const line: (value?: string) => FileSplitDelimiter;
37
+ const char: (value: string, keep?: FileSplitDelimiter["keep"]) => FileSplitDelimiter;
28
38
  }
29
- type BinaryWorkflow = Workflow<Type.TObject<{
30
- binary: Type.TSchema;
31
- }>, any>;
32
- export declare function split(binary: Binary, workflow: BinaryWorkflow, options?: FileSplitOptions & {
33
- parallel?: boolean;
34
- }): Promise<AsyncActivityAwaitable<StartWorkflowActivity>[]>;
35
- export {};
36
39
  //# sourceMappingURL=FileSplitActivity.d.ts.map