@t4h.framework/fs 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +10 -5
- package/.ai/skills/framework-fs-split/SKILL.md +113 -18
- package/README.md +120 -7
- package/dist/activities/FileJoinActivity.d.ts +32 -12
- package/dist/activities/FileJoinActivity.d.ts.map +1 -1
- package/dist/activities/FileJoinActivity.js +21 -34
- package/dist/activities/FileJoinActivity.js.map +1 -1
- package/dist/activities/FileReadActivity.d.ts +22 -0
- package/dist/activities/FileReadActivity.d.ts.map +1 -0
- package/dist/activities/FileReadActivity.js +68 -0
- package/dist/activities/FileReadActivity.js.map +1 -0
- package/dist/activities/FileSplitActivity.d.ts +2 -1
- package/dist/activities/FileSplitActivity.d.ts.map +1 -1
- package/dist/activities/FileSplitActivity.js +1 -0
- package/dist/activities/FileSplitActivity.js.map +1 -1
- package/dist/activities/__tests__/FileJoinActivity.spec.js +163 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.js.map +1 -1
- package/dist/activities/__tests__/FileReadActivity.spec.d.ts +2 -0
- package/dist/activities/__tests__/FileReadActivity.spec.d.ts.map +1 -0
- package/dist/activities/__tests__/FileReadActivity.spec.js +120 -0
- package/dist/activities/__tests__/FileReadActivity.spec.js.map +1 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.js +21 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.js.map +1 -1
- package/dist/constants/operations.d.ts +3 -0
- package/dist/constants/operations.d.ts.map +1 -0
- package/dist/constants/operations.js +18 -0
- package/dist/constants/operations.js.map +1 -0
- package/dist/fs.d.ts +1 -0
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +1 -0
- package/dist/fs.js.map +1 -1
- package/dist/helpers/CSV.d.ts +14 -0
- package/dist/helpers/CSV.d.ts.map +1 -0
- package/dist/helpers/CSV.js +44 -0
- package/dist/helpers/CSV.js.map +1 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +71 -2
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +1 -1
- package/dist/helpers/byte-delimiter-scanner.d.ts +8 -1
- package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -1
- package/dist/helpers/byte-delimiter-scanner.js +26 -2
- package/dist/helpers/byte-delimiter-scanner.js.map +1 -1
- package/dist/tools/assert-where-serializable.d.ts +2 -0
- package/dist/tools/assert-where-serializable.d.ts.map +1 -0
- package/dist/tools/assert-where-serializable.js +19 -0
- package/dist/tools/assert-where-serializable.js.map +1 -0
- package/package.json +4 -1
- package/dist/ActivityFileSystemClaim.d.ts +0 -7
- package/dist/ActivityFileSystemClaim.d.ts.map +0 -1
- package/dist/ActivityFileSystemClaim.js +0 -3
- package/dist/ActivityFileSystemClaim.js.map +0 -1
- package/dist/helpers/__tests__/csv-reader.spec.d.ts +0 -2
- package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +0 -1
- package/dist/helpers/__tests__/csv-reader.spec.js +0 -60
- package/dist/helpers/__tests__/csv-reader.spec.js.map +0 -1
- package/dist/helpers/csv-reader.d.ts +0 -8
- package/dist/helpers/csv-reader.d.ts.map +0 -1
- package/dist/helpers/csv-reader.js +0 -14
- package/dist/helpers/csv-reader.js.map +0 -1
- package/dist/helpers/parsers.d.ts +0 -9
- package/dist/helpers/parsers.d.ts.map +0 -1
- package/dist/helpers/parsers.js +0 -14
- package/dist/helpers/parsers.js.map +0 -1
- package/dist/testing/index.d.ts +0 -3
- package/dist/testing/index.d.ts.map +0 -1
- package/dist/testing/index.js +0 -32
- package/dist/testing/index.js.map +0 -1
|
@@ -24,10 +24,11 @@ Public exports live in `src/fs.ts`:
|
|
|
24
24
|
```typescript
|
|
25
25
|
import {
|
|
26
26
|
FileSystemClaim,
|
|
27
|
-
// file fan-out / fold — see the framework-fs-split skill
|
|
27
|
+
// file fan-out / fold / windows — see the framework-fs-split skill
|
|
28
28
|
split, Split, FileSplitActivity, type FileSplitOptions,
|
|
29
|
-
join, FileJoinActivity, type FileJoinOptions,
|
|
30
|
-
|
|
29
|
+
join, FileJoinActivity, type FileJoinOptions, type FileJoinWhere,
|
|
30
|
+
read, FileReadActivity, type FileReadOptions,
|
|
31
|
+
ByteDelimiterScanner, type FileSplitDelimiter, type ByteRange, type ByteTrim,
|
|
31
32
|
} from '@t4h.framework/fs'
|
|
32
33
|
```
|
|
33
34
|
|
|
@@ -61,7 +62,7 @@ export abstract class FileSystemClaim {
|
|
|
61
62
|
|--------|---------|
|
|
62
63
|
| `write(path, readable)` | Persist stream or buffer at `path`; return a `Binary` handle (metadata + `stream()`). |
|
|
63
64
|
| `read(path)` | Load an existing file at `path`; return a `Binary` handle. |
|
|
64
|
-
| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `split` children receive plain strings. |
|
|
65
|
+
| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `split` children receive plain strings. Implementations must **compose ranges**: slicing a slice is relative to it, and the stored range is absolute against the underlying file. |
|
|
65
66
|
|
|
66
67
|
All methods may return synchronously or as a `Promise`. Implementations choose storage layout; callers should only rely on the returned `Binary` API.
|
|
67
68
|
|
|
@@ -95,6 +96,10 @@ read the resulting `Binary` (e.g. `await res.body.json()` / `text()`). Your only
|
|
|
95
96
|
responsibility is that the runtime **registers a provider** (below). See
|
|
96
97
|
**Known consumers** for which package primitives to orchestrate.
|
|
97
98
|
|
|
99
|
+
This package's own workflow-facing primitives are `split`, `join` and `read`
|
|
100
|
+
(small byte windows — headers, trailers — hydrated into the workflow as a
|
|
101
|
+
string, capped at 1 MiB). See the `framework-fs-split` skill.
|
|
102
|
+
|
|
98
103
|
---
|
|
99
104
|
|
|
100
105
|
## Runtime provider
|
|
@@ -117,7 +122,7 @@ These packages depend on `@t4h.framework/fs` and declare `FileSystemClaim`; read
|
|
|
117
122
|
|---------|--------|
|
|
118
123
|
| `@t4h.framework/http` | The HTTP activity persists response bodies to `fs` and returns them wrapped in a `Body` — you read them with `res.body.json()` / `text()`. |
|
|
119
124
|
| `@t4h.framework/jsx` | Render activities write their output to `fs` and hand back a `Binary` you consume in the workflow. |
|
|
120
|
-
| `@t4h.framework/fs` itself | `FileSplitActivity` persists the durable copy and slices it by ranges; `FileJoinActivity` streams batch results into a new file. See the `framework-fs-split` skill. |
|
|
125
|
+
| `@t4h.framework/fs` itself | `FileSplitActivity` persists the durable copy and slices it by ranges; `FileJoinActivity` streams batch results into a new file; `FileReadActivity` slices (or streams, when the binary is not file-backed) a small window into a string. See the `framework-fs-split` skill. |
|
|
121
126
|
|
|
122
127
|
Orchestrate these package primitives from your workflow; the `fs` write/read happens inside them.
|
|
123
128
|
|
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: framework-fs-split
|
|
3
3
|
description: >-
|
|
4
|
-
Guides correct use of `split` and `
|
|
5
|
-
large file out into a batch of workflows (one per byte-range
|
|
6
|
-
folding the batch results back into a single file
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
4
|
+
Guides correct use of `split`, `join` and `read` from @t4h.framework/fs:
|
|
5
|
+
fanning one large file out into a batch of workflows (one per byte-range
|
|
6
|
+
slice), folding the batch results back into a single file, and reading small
|
|
7
|
+
byte windows (headers, trailers) into the workflow — with O(1) parent
|
|
8
|
+
history and bounded memory regardless of file size. Use whenever a workflow
|
|
9
|
+
must process a large file record by record — positional files with
|
|
10
|
+
header/trailer, plain-text lines, NDJSON, batch imports, per-record fan-out
|
|
11
|
+
with an aggregated output (e.g. a CSV report) — or when the user mentions
|
|
12
|
+
splitting/chunking a file, trimming a header/trailer, reading a file header,
|
|
13
|
+
filtering batch results, running a workflow per line/record, waiting for a
|
|
14
|
+
batch, collecting batch results, or the symbols `split`, `join`, `read`,
|
|
15
|
+
`Split`, `FileSplitActivity`, `FileJoinActivity`, `FileReadActivity`,
|
|
16
|
+
`WorkflowBatchRef`, `WorkflowBatchClaim`. Prefer this skill over
|
|
17
|
+
hand-rolling stream loops + `startWorkflow` calls.
|
|
15
18
|
---
|
|
16
19
|
|
|
17
|
-
# `split` / `join` — file fan-out and
|
|
20
|
+
# `split` / `join` / `read` — file fan-out, fold and windows for workflows
|
|
18
21
|
|
|
19
22
|
`split` turns **one file into a batch of workflows**: it streams the file once,
|
|
20
23
|
cuts it into **byte-range slices** (never copying slice content), starts one
|
|
@@ -23,7 +26,7 @@ terminal**. `join` is the inverse: it folds the batch results back into a
|
|
|
23
26
|
single file, streaming one result at a time.
|
|
24
27
|
|
|
25
28
|
```typescript
|
|
26
|
-
import { split, join, Split } from '@t4h.framework/fs'
|
|
29
|
+
import { read, split, join, Split } from '@t4h.framework/fs'
|
|
27
30
|
|
|
28
31
|
const processRecord = new Workflow(
|
|
29
32
|
{ id: 'process-record', schema: Type.String() },
|
|
@@ -34,9 +37,13 @@ const processRecord = new Workflow(
|
|
|
34
37
|
)
|
|
35
38
|
|
|
36
39
|
const main = new Workflow(fileSchema, async (file: Binary) => {
|
|
40
|
+
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
41
|
+
// header: string — recorded in history, never re-read on replay
|
|
42
|
+
|
|
37
43
|
const batch = await split(file, processRecord, {
|
|
38
44
|
delimiter: Split.char('\n10', { next: 2 }),
|
|
39
|
-
|
|
45
|
+
trim: { start: 241, end: 241 }, // header/trailer stay out of the batch
|
|
46
|
+
timeoutInMs: 3_600_000,
|
|
40
47
|
})
|
|
41
48
|
// this line only runs once the WHOLE batch is terminal;
|
|
42
49
|
// batch: WorkflowBatchRef — { id: string, count: number }
|
|
@@ -128,9 +135,10 @@ multi-byte encodings cannot skew the ranges a ranged download will use.
|
|
|
128
135
|
| Option | Default | Behavior |
|
|
129
136
|
|--------|---------|----------|
|
|
130
137
|
| `every` | `1` | Cut only every N delimiter occurrences — groups N records per child. One million lines with `every: 10_000` start 100 children, each receiving a raw block (inner delimiters untouched) to iterate locally. `keep` arithmetic applies only at actual cut points. |
|
|
138
|
+
| `trim` | none | `{ start?, end? }` **bytes** cut off each end of the file before scanning — a positional header/trailer. Byte-based on purpose: a record-start delimiter (`'\n10'`) fuses the trailer into the last record's slice, so a record-count skip would drop a valid record along with it. Trimmed content never becomes a slice nor counts toward `every`; fetch it with `read`. Positional layouts make the byte counts constants — getting them right is the caller's business. |
|
|
131
139
|
| `skipEmpty` | `true` | Drops zero-length slices (consecutive delimiters, file starting/ending with one) so they never become child workflows. Turn off when slice index must align 1:1 with record position. "Empty" means zero bytes — a whitespace-only line is a regular slice. |
|
|
132
140
|
| `encoding` | `'utf8'` | Converts the delimiter to bytes and tells the runtime how to hydrate each slice for the child (string when set on the slice). |
|
|
133
|
-
| `
|
|
141
|
+
| `timeoutInMs` | none | Max milliseconds to await the batch; on expiry the activity **rejects** (reason `timeout`) instead of suspending forever — the safety net for a child that never reaches a terminal state. |
|
|
134
142
|
|
|
135
143
|
---
|
|
136
144
|
|
|
@@ -150,7 +158,7 @@ Formats (content is the **child's** job — the child returns the final line;
|
|
|
150
158
|
|--------|----------|
|
|
151
159
|
| `lines` | Concatenates raw child outputs, one per line. Options: `header`, `separator` (default `'\n'`). Non-string outputs are JSON-stringified. |
|
|
152
160
|
| `entries` | NDJSON dump of `{ index, status, output }`, one per line — the format-agnostic view of the batch, rejected children included by default. |
|
|
153
|
-
| `csv` | Projects object outputs into RFC 4180 escaped columns. Options: `columns` (required; entries are `{ key, header? }` — `key` selects the value on the child output, `header` labels the header row, plain string is shorthand for `{ key }`), `delimiter` (default `','`), `header` (default `true`). Missing keys become empty cells; nested values are JSON inside the cell. Value *formatting* belongs in the child workflow (sandboxed user code); the join only owns layout. |
|
|
161
|
+
| `csv` | Projects object outputs into RFC 4180 escaped columns. Options: `columns` (required; entries are `{ key, header? }` — `key` selects the value on the child output (a dot-path like `'meta.total'` reaches nested fields, matching the `where` semantics; a literal top-level key with dots wins over the nested path), `header` labels the header row, plain string is shorthand for `{ key }`), `delimiter` (default `','`), `header` (default `true`). Missing keys become empty cells; nested values selected whole are JSON inside the cell. Value *formatting* belongs in the child workflow (sandboxed user code); the join only owns layout. |
|
|
154
162
|
|
|
155
163
|
**Status filter** (`status?: ('fulfilled' | 'rejected')[]`) selects which
|
|
156
164
|
results make it into the file — there is no error policy. Defaults:
|
|
@@ -161,14 +169,92 @@ design; a reprocessing file of failures is one call away:
|
|
|
161
169
|
const failures = await join(batch, { format: 'entries', status: ['rejected'] })
|
|
162
170
|
```
|
|
163
171
|
|
|
172
|
+
**Where filter** (`where?`) narrows results with a **sift (MongoDB-style)
|
|
173
|
+
query** over each entry's output, applied after `status`. Whitelisted subset
|
|
174
|
+
only — declarative, JSON-only operators: `$eq`, `$ne`, `$lt`, `$lte`, `$gt`,
|
|
175
|
+
`$gte`, `$in`, `$nin`, `$exists`, `$not`, `$and`, `$or`, `$nor`, plus plain
|
|
176
|
+
equality and dot-paths for nested fields. Pass the child output type as a
|
|
177
|
+
generic to narrow top-level keys/values (dot-paths accepted, not narrowed):
|
|
178
|
+
|
|
179
|
+
```typescript
|
|
180
|
+
const relevant = await join<InvoiceResult>(batch, {
|
|
181
|
+
format: 'csv',
|
|
182
|
+
columns: ['id', 'total'],
|
|
183
|
+
where: { valid: true, total: { $gt: 100 }, 'meta.situacao': 'aprovada' },
|
|
184
|
+
})
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Non-serializable filters are rejected eagerly at `toInput` (`TypeError`):
|
|
188
|
+
functions (incl. `$where`), `RegExp` (incl. `$regex`), `undefined`, operators
|
|
189
|
+
outside the subset — a dropped closure would make the replay filter
|
|
190
|
+
differently than the first execution. Arbitrary predicates belong in a custom
|
|
191
|
+
activity in project code (the function then lives in versioned code).
|
|
192
|
+
MongoDB semantics apply: array fields match by containment (`{ tags: 'a' }`
|
|
193
|
+
matches `['a', 'b']`), object values compare deeply, no type coercion. The
|
|
194
|
+
query compiles once and runs inside the existing streaming loop over
|
|
195
|
+
already-hydrated outputs — same memory profile; app-side (decides what enters
|
|
196
|
+
the file, not what leaves the database).
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## `read`
|
|
201
|
+
|
|
202
|
+
```typescript
|
|
203
|
+
async function read(
|
|
204
|
+
binary: Binary,
|
|
205
|
+
options?: { range?: { start: number; end: number }; encoding?: BufferEncoding },
|
|
206
|
+
): Promise<string>
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Reads a byte window of a `Binary` into the workflow as a decoded string. The
|
|
210
|
+
output is recorded in history (replay never re-reads the file), so reads are
|
|
211
|
+
**capped at 1 MiB** — this is the primitive for headers, trailers and single
|
|
212
|
+
records; a data file flows through `split`, never through `read`. `range`
|
|
213
|
+
defaults to the whole binary (same cap applies); `encoding` defaults to
|
|
214
|
+
`'utf8'`.
|
|
215
|
+
|
|
216
|
+
Two paths under the hood:
|
|
217
|
+
|
|
218
|
+
1. **Fast path** — `FileSystemClaim.slice`: on a file-backed binary a trailer
|
|
219
|
+
read is a ranged download, whatever the file size.
|
|
220
|
+
2. **Streaming fallback** — binaries the claim cannot slice (e.g. an SFTP
|
|
221
|
+
reference): the stream is consumed up to the window and destroyed as soon
|
|
222
|
+
as it is filled. A header read pulls the first chunks only, whatever the
|
|
223
|
+
file size; a **trailing window over a non-sliceable binary** is the one
|
|
224
|
+
expensive case — the stream traverses the whole file to reach it.
|
|
225
|
+
|
|
226
|
+
Reading a slice is **relative to the slice** (`range: { start: 0, end: 10 }`
|
|
227
|
+
returns its first 10 bytes, not the underlying file's) — slice ranges compose.
|
|
228
|
+
|
|
229
|
+
The positional-file recipe — header/trailer in the parent, records in the
|
|
230
|
+
batch:
|
|
231
|
+
|
|
232
|
+
```typescript
|
|
233
|
+
const header = await read(file, { range: { start: 0, end: HEADER } })
|
|
234
|
+
const trailer = await read(file, {
|
|
235
|
+
range: { start: file.contentLength - TRAILER, end: file.contentLength },
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
const batch = await split(file, processInvoice, {
|
|
239
|
+
delimiter: Split.char('\n10', { next: 2 }),
|
|
240
|
+
trim: { start: HEADER + 1, end: TRAILER + 1 }, // +1: boundary newlines
|
|
241
|
+
})
|
|
242
|
+
|
|
243
|
+
// e.g. validate the trailer's record count against batch.count
|
|
244
|
+
```
|
|
245
|
+
|
|
164
246
|
---
|
|
165
247
|
|
|
166
248
|
## Runtime requirements
|
|
167
249
|
|
|
168
250
|
- Claim providers: `FileSystemClaim` (with `slice`), `WorkflowClaim` (with
|
|
169
251
|
`startMany`), and `WorkflowBatchClaim` (from `@t4h.framework/core`).
|
|
170
|
-
- Production whitelist: `FileSplitActivity`, `FileJoinActivity
|
|
171
|
-
`StartManyWorkflowsActivity` must be
|
|
252
|
+
- Production whitelist: `FileSplitActivity`, `FileJoinActivity`,
|
|
253
|
+
`FileReadActivity` and core's `StartManyWorkflowsActivity` must be
|
|
254
|
+
registered via `runtime.addActivity`.
|
|
255
|
+
- `FileSystemClaim.slice` implementations must **compose ranges** when slicing
|
|
256
|
+
an already-sliced binary (relative in, absolute against the underlying file
|
|
257
|
+
out) — `read` and slice consumers rely on that semantics.
|
|
172
258
|
- `FileSplitActivity` is an **AsyncActivity**: the heavy work happens in
|
|
173
259
|
`bootstrap`, executed worker side atomically with the pending suspension —
|
|
174
260
|
children cannot finish before the parent is awaiting them.
|
|
@@ -240,6 +326,15 @@ CSV escaping (quotes doubled, delimiter/newline wrapped); the status filter.
|
|
|
240
326
|
intact (hash it, upload it), pass the `Binary` through.
|
|
241
327
|
- **Forgetting `keep` on positional delimiters.** `Split.char('\n10')` without
|
|
242
328
|
`{ next: 2 }` silently discards the record type code from every slice.
|
|
329
|
+
- **Splitting a positional file without `trim`.** The header rides inside the
|
|
330
|
+
first slice (with `every: N` it silently consumes a record slot) and the
|
|
331
|
+
trailer fuses into the last one. Trim both ends; fetch them with `read`.
|
|
332
|
+
- **Reading a data file with `read`.** The output lands in workflow history;
|
|
333
|
+
the 1 MiB cap is a guard, not a target. Records flow through `split`.
|
|
334
|
+
- **Passing a function as a join filter.** Options serialize into history — a
|
|
335
|
+
closure cannot survive it, so `toInput` rejects functions, `$where`,
|
|
336
|
+
`RegExp` and out-of-subset operators with a `TypeError`. Use the `where`
|
|
337
|
+
sift subset or a custom activity whose predicate lives in versioned code.
|
|
243
338
|
- **Hand-rolling the loop.** A stream-and-`startWorkflow` loop inside an
|
|
244
339
|
activity re-parses on replay, misses the barrier, and pushes N entries into
|
|
245
340
|
history.
|
package/README.md
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# @t4h.framework/fs
|
|
2
2
|
|
|
3
|
-
File system claim for reading and writing files within activities,
|
|
4
|
-
|
|
3
|
+
File system claim for reading and writing files within activities, plus the
|
|
4
|
+
workflow-facing file activities: `split` fans a large file out into one child
|
|
5
|
+
workflow per slice, `join` folds the batch results back into a single file,
|
|
6
|
+
and `read` brings a small byte window (a header, a trailer) into the workflow
|
|
7
|
+
as a string.
|
|
5
8
|
|
|
6
9
|
## Claim
|
|
7
10
|
|
|
@@ -69,6 +72,41 @@ The runtime must supply a concrete implementation:
|
|
|
69
72
|
{ provide: FileSystemClaim, value: new LocalFileSystemClaimImpl() }
|
|
70
73
|
```
|
|
71
74
|
|
|
75
|
+
## File read
|
|
76
|
+
|
|
77
|
+
### read
|
|
78
|
+
|
|
79
|
+
Reads a byte window of a `Binary` into the workflow as a decoded string. The
|
|
80
|
+
output is recorded in the workflow history (replay never re-reads the file),
|
|
81
|
+
which is why reads are capped at 1 MiB: this is the primitive for headers,
|
|
82
|
+
trailers and single records — a data file flows through `split`, never through
|
|
83
|
+
`read`.
|
|
84
|
+
|
|
85
|
+
```typescript
|
|
86
|
+
import { read } from '@t4h.framework/fs'
|
|
87
|
+
|
|
88
|
+
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
89
|
+
|
|
90
|
+
const trailer = await read(file, {
|
|
91
|
+
range: { start: file.contentLength - 240, end: file.contentLength },
|
|
92
|
+
})
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Options: `range` (default: the whole binary, subject to the same cap) and
|
|
96
|
+
`encoding` (default `'utf8'`).
|
|
97
|
+
|
|
98
|
+
The activity tries `FileSystemClaim.slice` first — on a file-backed binary a
|
|
99
|
+
trailer read is a ranged download, whatever the file size. Binaries the claim
|
|
100
|
+
cannot slice (e.g. an SFTP reference) fall back to streaming: the stream is
|
|
101
|
+
consumed up to the window and destroyed as soon as it is filled, so a header
|
|
102
|
+
read pulls the first chunks only, whatever the file size. The one expensive
|
|
103
|
+
case is a trailing window over a non-sliceable binary — the stream must
|
|
104
|
+
traverse the whole file to reach it.
|
|
105
|
+
|
|
106
|
+
Reading a slice is relative to the slice: `read(slice, { range: { start: 0,
|
|
107
|
+
end: 10 } })` returns the first 10 bytes of the slice, not of the underlying
|
|
108
|
+
file — slice ranges compose.
|
|
109
|
+
|
|
72
110
|
## File split
|
|
73
111
|
|
|
74
112
|
### split
|
|
@@ -148,9 +186,10 @@ start of the next slice.
|
|
|
148
186
|
await split(file, workflow, {
|
|
149
187
|
delimiter: Split.line(), // required: where to cut
|
|
150
188
|
every: 10_000, // cut only every N occurrences (groups records)
|
|
189
|
+
trim: { start: 241, end: 241 }, // bytes cut off each end before scanning
|
|
151
190
|
skipEmpty: false, // keep zero-length slices (default: skipped)
|
|
152
191
|
encoding: 'latin1', // delimiter bytes + slice hydration (default: utf8)
|
|
153
|
-
|
|
192
|
+
timeoutInMs: 3_600_000, // max ms to await the batch; rejects on expiry
|
|
154
193
|
})
|
|
155
194
|
```
|
|
156
195
|
|
|
@@ -158,6 +197,23 @@ await split(file, workflow, {
|
|
|
158
197
|
10_000` starts 100 children instead of one million; each child receives a
|
|
159
198
|
raw block with its inner delimiters untouched and iterates locally. `keep`
|
|
160
199
|
arithmetic only applies at actual cut points.
|
|
200
|
+
- **`trim`** cuts a fixed number of bytes off each end of the file before
|
|
201
|
+
scanning — a positional header/trailer, which may not be aligned with the
|
|
202
|
+
record delimiter (a record-start delimiter like `Split.char('\n10')` fuses
|
|
203
|
+
the trailer into the last record's slice, so counting records would drop a
|
|
204
|
+
valid one along with it). Trimmed content never becomes a slice nor counts
|
|
205
|
+
toward `every`; fetch it with `read`. Getting the byte counts right is the
|
|
206
|
+
caller's business — positional layouts make them constants:
|
|
207
|
+
|
|
208
|
+
```typescript
|
|
209
|
+
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
210
|
+
|
|
211
|
+
const batch = await split(file, processInvoice, {
|
|
212
|
+
delimiter: Split.char('\n10', { next: 2 }),
|
|
213
|
+
trim: { start: 241, end: 241 }, // header + '\n', trailer + '\n'
|
|
214
|
+
})
|
|
215
|
+
```
|
|
216
|
+
|
|
161
217
|
- **`skipEmpty`** (default `true`) drops zero-length slices — consecutive
|
|
162
218
|
delimiters, or a file starting/ending with one — so they never become child
|
|
163
219
|
workflows. Turn it off when the slice index must align 1:1 with the record
|
|
@@ -166,7 +222,7 @@ await split(file, workflow, {
|
|
|
166
222
|
- **`encoding`** converts the delimiter to bytes and tells the runtime how to
|
|
167
223
|
hydrate each slice for the child. Offsets are always computed over raw
|
|
168
224
|
bytes, so multi-byte encodings never skew the ranges.
|
|
169
|
-
- **`
|
|
225
|
+
- **`timeoutInMs`** is the maximum time in milliseconds to await the batch. On
|
|
170
226
|
expiry the activity rejects (reason `timeout`) instead of suspending
|
|
171
227
|
forever — the safety net for a child that never reaches a terminal state.
|
|
172
228
|
|
|
@@ -203,9 +259,12 @@ const csv = await join(batch, {
|
|
|
203
259
|
default.
|
|
204
260
|
- **`csv`** — projects object outputs into RFC 4180 escaped columns. Options:
|
|
205
261
|
`columns` (required; each entry is `{ key, header? }` where `key` selects
|
|
206
|
-
the value on the child output
|
|
207
|
-
|
|
208
|
-
|
|
262
|
+
the value on the child output — a dot-path like `'meta.total'` reaches
|
|
263
|
+
nested fields, matching the `where` semantics; a literal top-level key that
|
|
264
|
+
contains dots wins over the nested path — and `header` is the label on the
|
|
265
|
+
header row — a plain string is shorthand for `{ key }`), `delimiter`
|
|
266
|
+
(default `','`), `header` (default `true`; emits the column labels as the
|
|
267
|
+
first row).
|
|
209
268
|
|
|
210
269
|
### Status filter
|
|
211
270
|
|
|
@@ -218,6 +277,60 @@ rejected children for reprocessing is one call away:
|
|
|
218
277
|
const failures = await join(batch, { format: 'entries', status: ['rejected'] })
|
|
219
278
|
```
|
|
220
279
|
|
|
280
|
+
### Where filter
|
|
281
|
+
|
|
282
|
+
`where` narrows results with a [sift](https://github.com/crcn/sift.js)
|
|
283
|
+
(MongoDB-style) query over each entry's output, applied after the `status`
|
|
284
|
+
filter. The accepted subset is deliberate — only declarative, JSON-only
|
|
285
|
+
operators: `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`, `$nin`,
|
|
286
|
+
`$exists`, `$not`, `$and`, `$or`, `$nor`, plus plain equality and dot-paths
|
|
287
|
+
for nested fields:
|
|
288
|
+
|
|
289
|
+
```typescript
|
|
290
|
+
const relevant = await join(batch, {
|
|
291
|
+
format: 'csv',
|
|
292
|
+
columns: ['id', 'total'],
|
|
293
|
+
where: {
|
|
294
|
+
valid: true, // plain strict equality
|
|
295
|
+
total: { $gt: 100 }, // operators
|
|
296
|
+
'meta.situacao': 'aprovada', // dot-path into nested output
|
|
297
|
+
$or: [{ ref: 'a' }, { ref: 'b' }],
|
|
298
|
+
},
|
|
299
|
+
})
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
`join` also accepts the child output type as a generic — top-level keys then
|
|
303
|
+
narrow to its fields, with values typed by the field (dot-paths are accepted
|
|
304
|
+
without narrowing):
|
|
305
|
+
|
|
306
|
+
```typescript
|
|
307
|
+
type InvoiceResult = { id: string; valid: boolean; total: number }
|
|
308
|
+
|
|
309
|
+
await join<InvoiceResult>(batch, {
|
|
310
|
+
format: 'csv',
|
|
311
|
+
columns: ['id'],
|
|
312
|
+
where: { valid: true, total: { $gt: 100 } }, // { valid: 'yes' } rejects
|
|
313
|
+
})
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
Anything that cannot serialize losslessly into the workflow history is
|
|
317
|
+
rejected eagerly at `toInput` with a `TypeError`: functions (including
|
|
318
|
+
`$where`), `RegExp` values (including `$regex`), `undefined`, and operators
|
|
319
|
+
outside the subset — a dropped closure would make the replay filter
|
|
320
|
+
differently than the first execution. For arbitrary predicates, write a
|
|
321
|
+
custom activity in project code (the function then lives in versioned code,
|
|
322
|
+
not in history).
|
|
323
|
+
|
|
324
|
+
Two MongoDB semantics worth knowing: array fields match by containment
|
|
325
|
+
(`{ tags: 'a' }` matches `tags: ['a', 'b']`), and equality against an object
|
|
326
|
+
value compares deeply (never by reference). Comparisons never coerce types.
|
|
327
|
+
|
|
328
|
+
Filtering runs inside the existing streaming loop, over outputs that are
|
|
329
|
+
already hydrated one at a time — the query compiles once and adds one
|
|
330
|
+
predicate call per entry, no extra serialization and no change to the memory
|
|
331
|
+
profile. It is applied app-side: every child output is still fetched and
|
|
332
|
+
parsed; `where` decides what enters the file, not what leaves the database.
|
|
333
|
+
|
|
221
334
|
### ByteDelimiterScanner
|
|
222
335
|
|
|
223
336
|
The scanner behind `split` is exported for direct use. It is a pass-through
|
|
@@ -1,27 +1,47 @@
|
|
|
1
|
-
import { type Binary, SyncActivity, type WorkflowBatchRef } from '@t4h.framework/core';
|
|
1
|
+
import { type Binary, type Serializable, SyncActivity, type WorkflowBatchRef } from '@t4h.framework/core';
|
|
2
|
+
import { CSV } from '../helpers/CSV.js';
|
|
2
3
|
export type FileJoinStatus = 'fulfilled' | 'rejected';
|
|
3
|
-
type
|
|
4
|
+
type FileJoinWhereOperators<TValue> = {
|
|
5
|
+
$eq?: TValue;
|
|
6
|
+
$ne?: TValue;
|
|
7
|
+
$lt?: TValue;
|
|
8
|
+
$lte?: TValue;
|
|
9
|
+
$gt?: TValue;
|
|
10
|
+
$gte?: TValue;
|
|
11
|
+
$in?: TValue[];
|
|
12
|
+
$nin?: TValue[];
|
|
13
|
+
$exists?: boolean;
|
|
14
|
+
$not?: FileJoinWhereCondition<TValue>;
|
|
15
|
+
};
|
|
16
|
+
type FileJoinWhereCondition<TValue> = TValue | FileJoinWhereOperators<TValue>;
|
|
17
|
+
export type FileJoinWhere<TOutput = unknown> = ([TOutput] extends [object] ? {
|
|
18
|
+
[K in keyof TOutput]?: FileJoinWhereCondition<Exclude<TOutput[K], undefined>>;
|
|
19
|
+
} & {
|
|
20
|
+
[path: `${string}.${string}`]: Serializable;
|
|
21
|
+
} : Record<string, Serializable>) & {
|
|
22
|
+
$and?: FileJoinWhere<TOutput>[];
|
|
23
|
+
$or?: FileJoinWhere<TOutput>[];
|
|
24
|
+
$nor?: FileJoinWhere<TOutput>[];
|
|
25
|
+
};
|
|
26
|
+
type FileJoinOptionsBase<TOutput> = {
|
|
4
27
|
status?: FileJoinStatus[];
|
|
28
|
+
where?: FileJoinWhere<TOutput>;
|
|
5
29
|
};
|
|
6
|
-
type FileJoinOptionsLines = FileJoinOptionsBase & {
|
|
30
|
+
type FileJoinOptionsLines<TOutput> = FileJoinOptionsBase<TOutput> & {
|
|
7
31
|
format: 'lines';
|
|
8
32
|
header?: string;
|
|
9
33
|
separator?: string;
|
|
10
34
|
};
|
|
11
|
-
type FileJoinOptionsEntries = FileJoinOptionsBase & {
|
|
35
|
+
type FileJoinOptionsEntries<TOutput> = FileJoinOptionsBase<TOutput> & {
|
|
12
36
|
format: 'entries';
|
|
13
37
|
};
|
|
14
|
-
|
|
15
|
-
key: string;
|
|
16
|
-
header?: string;
|
|
17
|
-
};
|
|
18
|
-
type FileJoinOptionsCsv = FileJoinOptionsBase & {
|
|
38
|
+
type FileJoinOptionsCsv<TOutput> = FileJoinOptionsBase<TOutput> & {
|
|
19
39
|
format: 'csv';
|
|
20
|
-
columns:
|
|
40
|
+
columns: CSV.Column[];
|
|
21
41
|
delimiter?: string;
|
|
22
42
|
header?: boolean;
|
|
23
43
|
};
|
|
24
|
-
export type FileJoinOptions = FileJoinOptionsLines | FileJoinOptionsEntries | FileJoinOptionsCsv
|
|
44
|
+
export type FileJoinOptions<TOutput = unknown> = FileJoinOptionsLines<TOutput> | FileJoinOptionsEntries<TOutput> | FileJoinOptionsCsv<TOutput>;
|
|
25
45
|
export type FileJoinActivityInput = {
|
|
26
46
|
batch: string;
|
|
27
47
|
options: FileJoinOptions;
|
|
@@ -34,6 +54,6 @@ export declare class FileJoinActivity extends SyncActivity<readonly [batch: Work
|
|
|
34
54
|
private writeHeader;
|
|
35
55
|
private toLine;
|
|
36
56
|
}
|
|
37
|
-
export declare function join(batch: WorkflowBatchRef, options: FileJoinOptions): Promise<Binary>;
|
|
57
|
+
export declare function join<TOutput = unknown>(batch: WorkflowBatchRef, options: FileJoinOptions<TOutput>): Promise<Binary>;
|
|
38
58
|
export {};
|
|
39
59
|
//# sourceMappingURL=FileJoinActivity.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,YAAY,EAGZ,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAA;
|
|
1
|
+
{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,KAAK,YAAY,EACjB,YAAY,EAGZ,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAGvC,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,CAAA;AAErD,KAAK,sBAAsB,CAAC,MAAM,IAAI;IACpC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,EAAE,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,IAAI,CAAC,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAA;CACtC,CAAA;AAED,KAAK,sBAAsB,CAAC,MAAM,IAAI,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAA;AAE7E,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,GACtE;KACG,CAAC,IAAI,MAAM,OAAO,CAAC,CAAC,EAAE,sBAAsB,CAC3C,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAC/B;CACF,GAAG;IACF,CAAC,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,GAAG,YAAY,CAAA;CAC5C,GACD,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,GAAG;IAClC,IAAI,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;IAC/B,GAAG,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;IAC9B,IAAI,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAA;CAChC,CAAA;AAED,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAQD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,eAAe,CAAC,EAC5D,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAMX,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAsCrE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,WAAW;IAazB,OAAO,CAAC,MAAM;CAef;AAED,wBAAsB,IAAI,CAAC,OAAO,GAAG,OAAO,EAC1C,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC,CAMjB"}
|
|
@@ -1,35 +1,24 @@
|
|
|
1
1
|
import { once } from 'node:events';
|
|
2
2
|
import { PassThrough } from 'node:stream';
|
|
3
3
|
import { Claim, History, SyncActivity, WorkflowBatchClaim, } from '@t4h.framework/core';
|
|
4
|
+
import sift from 'sift';
|
|
5
|
+
import { OPERATIONS } from '../constants/operations.js';
|
|
4
6
|
import { FileSystemClaim } from '../FileSystemClaim.js';
|
|
7
|
+
import { CSV } from '../helpers/CSV.js';
|
|
8
|
+
import { assertWhereSerializable } from '../tools/assert-where-serializable.js';
|
|
5
9
|
const extensions = {
|
|
6
10
|
lines: 'txt',
|
|
7
11
|
entries: 'ndjson',
|
|
8
12
|
csv: 'csv',
|
|
9
13
|
};
|
|
10
|
-
function columnKey(column) {
|
|
11
|
-
return typeof column === 'string' ? column : column.key;
|
|
12
|
-
}
|
|
13
|
-
function columnHeader(column) {
|
|
14
|
-
return typeof column === 'string' ? column : (column.header ?? column.key);
|
|
15
|
-
}
|
|
16
|
-
function csvEscape(value, delimiter) {
|
|
17
|
-
if (value === null || typeof value === 'undefined')
|
|
18
|
-
return '';
|
|
19
|
-
const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
|
|
20
|
-
if (text.includes(delimiter) ||
|
|
21
|
-
text.includes('"') ||
|
|
22
|
-
text.includes('\n') ||
|
|
23
|
-
text.includes('\r'))
|
|
24
|
-
return `"${text.replaceAll('"', '""')}"`;
|
|
25
|
-
return text;
|
|
26
|
-
}
|
|
27
14
|
export class FileJoinActivity extends SyncActivity {
|
|
28
15
|
claims = new Claim({
|
|
29
16
|
fs: FileSystemClaim,
|
|
30
17
|
batch: WorkflowBatchClaim,
|
|
31
18
|
});
|
|
32
19
|
toInput(batch, options) {
|
|
20
|
+
if (options.where)
|
|
21
|
+
assertWhereSerializable(options.where, 'where');
|
|
33
22
|
return { batch: batch.id, options };
|
|
34
23
|
}
|
|
35
24
|
async run({ batch, options }) {
|
|
@@ -39,14 +28,19 @@ export class FileJoinActivity extends SyncActivity {
|
|
|
39
28
|
if (!stream.write(line))
|
|
40
29
|
await once(stream, 'drain');
|
|
41
30
|
};
|
|
42
|
-
const statuses = new Set(options.status ??
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
31
|
+
const statuses = new Set(options.status ?? ['fulfilled']);
|
|
32
|
+
if (!options.status && options.format === 'entries')
|
|
33
|
+
statuses.add('rejected');
|
|
34
|
+
const matches = options.where &&
|
|
35
|
+
sift.createQueryTester(options.where, {
|
|
36
|
+
operations: OPERATIONS,
|
|
37
|
+
});
|
|
46
38
|
await this.writeHeader(options, write);
|
|
47
39
|
for await (const entry of this.claims.batch.outputs(batch)) {
|
|
48
40
|
if (!statuses.has(entry.status))
|
|
49
41
|
continue;
|
|
42
|
+
if (matches && !matches(entry.output))
|
|
43
|
+
continue;
|
|
50
44
|
await write(this.toLine(entry, options));
|
|
51
45
|
}
|
|
52
46
|
stream.end();
|
|
@@ -58,23 +52,16 @@ export class FileJoinActivity extends SyncActivity {
|
|
|
58
52
|
async writeHeader(options, write) {
|
|
59
53
|
if (options.format === 'lines' && options.header)
|
|
60
54
|
await write(options.header + (options.separator ?? '\n'));
|
|
61
|
-
if (options.format === 'csv' && options.header !== false)
|
|
62
|
-
|
|
63
|
-
await write(options.columns
|
|
64
|
-
.map(column => csvEscape(columnHeader(column), delimiter))
|
|
65
|
-
.join(delimiter) + '\n');
|
|
66
|
-
}
|
|
55
|
+
if (options.format === 'csv' && options.header !== false)
|
|
56
|
+
await write(CSV.toHeader(options.columns, { delimiter: options.delimiter }));
|
|
67
57
|
}
|
|
68
58
|
toLine(entry, options) {
|
|
69
59
|
if (options.format === 'entries')
|
|
70
60
|
return JSON.stringify(entry) + '\n';
|
|
71
|
-
if (options.format === 'csv')
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
.map(column => csvEscape(output[columnKey(column)], delimiter))
|
|
76
|
-
.join(delimiter) + '\n');
|
|
77
|
-
}
|
|
61
|
+
if (options.format === 'csv')
|
|
62
|
+
return CSV.toLine(entry.output, options.columns, {
|
|
63
|
+
delimiter: options.delimiter,
|
|
64
|
+
});
|
|
78
65
|
const value = typeof entry.output === 'string'
|
|
79
66
|
? entry.output
|
|
80
67
|
: JSON.stringify(entry.output);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,
|
|
1
|
+
{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,EAEP,YAAY,EACZ,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAA;AAiE/E,MAAM,UAAU,GAA8C;IAC5D,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,QAAQ;IACjB,GAAG,EAAE,KAAK;CACX,CAAA;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,KAAuB,EACvB,OAAwB;QAExB,IAAI,OAAO,CAAC,KAAK;YAAE,uBAAuB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAElE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;IACrC,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAyB;QACxD,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAA;QAEhC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAClC,QAAQ,IAAI,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC/C,MAAM,CACP,CAAA;QAED,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,CAAC,CAAA;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAiB,OAAO,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YACjD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAE1B,MAAM,OAAO,GACX,OAAO,CAAC,KAAK;YACb,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,EAAE;gBACpC,UAAU,EAAE,UAAU;aACvB,CAAC,CAAA;QAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAEtC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAEzC,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAE/C,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;QAC1C,CAAC;QAED,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAwB,EACxB,KAAsC;QAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM;YAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAA;QAE3D,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YACtD,MAAM,KAAK,CACT,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAChE,CAAA;IACL,CAAC;IAEO,MAAM,CAAC,KAA0B,EAAE,OAAwB;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;QAErE,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YAC1B,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAA;QAEJ,MAAM,KAAK,GACT,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;YAC9B,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAElC,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAA;IAC5C,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,KAAuB,EACvB,OAAiC;IAEjC,OAAO,MAAM,OAAO,CAAC,UAAU,CAC7B,gBAAgB,EAChB,KAAK,EACL,OAA0B,CAC3B,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type Binary, SyncActivity } from '@t4h.framework/core';
|
|
2
|
+
export type FileReadOptions = {
|
|
3
|
+
range?: {
|
|
4
|
+
start: number;
|
|
5
|
+
end: number;
|
|
6
|
+
};
|
|
7
|
+
encoding?: BufferEncoding;
|
|
8
|
+
};
|
|
9
|
+
export type FileReadActivityInput = {
|
|
10
|
+
binary: Binary;
|
|
11
|
+
options: FileReadOptions;
|
|
12
|
+
};
|
|
13
|
+
export declare class FileReadActivity extends SyncActivity<readonly [binary: Binary, options: FileReadOptions], FileReadActivityInput, string, string> {
|
|
14
|
+
private readonly claims;
|
|
15
|
+
toInput(binary: Binary, options: FileReadOptions): FileReadActivityInput;
|
|
16
|
+
run({ binary, options, }: FileReadActivityInput): Promise<string>;
|
|
17
|
+
toOutput(output: string): string;
|
|
18
|
+
private slice;
|
|
19
|
+
private window;
|
|
20
|
+
}
|
|
21
|
+
export declare function read(binary: Binary, options?: FileReadOptions): Promise<string>;
|
|
22
|
+
//# sourceMappingURL=FileReadActivity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAID,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;CAuCrB;AAED,wBAAsB,IAAI,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAEjB"}
|