@t4h.framework/fs 0.12.2 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +25 -26
- package/.ai/skills/framework-fs-split/SKILL.md +40 -54
- package/.ai/skills/framework-fs-write/SKILL.md +13 -15
- package/CHANGELOG.md +44 -0
- package/README.md +14 -41
- package/dist/activities/FileJoinActivity.d.ts +3 -3
- package/dist/activities/FileJoinActivity.d.ts.map +1 -1
- package/dist/activities/FileJoinActivity.js.map +1 -1
- package/dist/activities/FileReadActivity.d.ts +1 -0
- package/dist/activities/FileReadActivity.d.ts.map +1 -1
- package/dist/activities/FileReadActivity.js +24 -19
- package/dist/activities/FileReadActivity.js.map +1 -1
- package/dist/activities/FileSplitActivity.d.ts +8 -15
- package/dist/activities/FileSplitActivity.d.ts.map +1 -1
- package/dist/activities/FileSplitActivity.js +18 -32
- package/dist/activities/FileSplitActivity.js.map +1 -1
- package/dist/constants/operations.d.ts +1 -1
- package/dist/constants/operations.d.ts.map +1 -1
- package/dist/constants/operations.js.map +1 -1
- package/dist/fs.d.ts +1 -1
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +0 -1
- package/dist/fs.js.map +1 -1
- package/dist/helpers/CSV.d.ts.map +1 -1
- package/dist/helpers/CSV.js +3 -1
- package/dist/helpers/CSV.js.map +1 -1
- package/dist/helpers/group-records.d.ts +12 -0
- package/dist/helpers/group-records.d.ts.map +1 -0
- package/dist/helpers/group-records.js +36 -0
- package/dist/helpers/group-records.js.map +1 -0
- package/dist/helpers/split-records.d.ts +30 -0
- package/dist/helpers/split-records.d.ts.map +1 -0
- package/dist/helpers/split-records.js +122 -0
- package/dist/helpers/split-records.js.map +1 -0
- package/package.json +4 -3
- package/dist/helpers/ByteDelimiterScanner.d.ts +0 -50
- package/dist/helpers/ByteDelimiterScanner.d.ts.map +0 -1
- package/dist/helpers/ByteDelimiterScanner.js +0 -103
- package/dist/helpers/ByteDelimiterScanner.js.map +0 -1
- package/dist/helpers/ByteRecordFilter.d.ts +0 -32
- package/dist/helpers/ByteRecordFilter.d.ts.map +0 -1
- package/dist/helpers/ByteRecordFilter.js +0 -89
- package/dist/helpers/ByteRecordFilter.js.map +0 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: framework-fs
|
|
3
3
|
description: >-
|
|
4
4
|
Guides correct use of @t4h.framework/fs in T4H Framework workflows: the
|
|
5
|
-
FileSystemClaim abstraction that the package activities (http,
|
|
5
|
+
FileSystemClaim abstraction that the package activities (http, pdf) use to
|
|
6
6
|
read/write files, the Binary result workflows consume, and runtime providers.
|
|
7
7
|
A workflow never implements Binary itself — see the framework-fs-write skill.
|
|
8
8
|
Use when a workflow orchestrates file-backed package activities, consuming
|
|
@@ -31,12 +31,9 @@ import {
|
|
|
31
31
|
read, FileReadActivity, type FileReadOptions,
|
|
32
32
|
// turn workflow content into a Binary — see the framework-fs-write skill
|
|
33
33
|
write, FileWriteActivity, type FileWriteOptions,
|
|
34
|
-
ByteDelimiterScanner, type FileSplitDelimiter, type ByteRange, type ByteTrim,
|
|
35
34
|
} from '@t4h.framework/fs'
|
|
36
35
|
```
|
|
37
36
|
|
|
38
|
-
Do not import from internal paths such as `src/FileSystemClaim.ts` in application code.
|
|
39
|
-
|
|
40
37
|
---
|
|
41
38
|
|
|
42
39
|
## FileSystemClaim contract
|
|
@@ -66,7 +63,7 @@ export abstract class FileSystemClaim {
|
|
|
66
63
|
|--------|---------|
|
|
67
64
|
| `write(path, readable, contentType?)` | Persist stream or buffer at `path`; return a `Binary` handle (metadata + `stream()`). When `contentType` is omitted, infer it from the path. |
|
|
68
65
|
| `read(path)` | Load an existing file at `path`; return a `Binary` handle. |
|
|
69
|
-
| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `
|
|
66
|
+
| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `read` returns a window as a string. Implementations must **compose ranges**: slicing a slice is relative to it, and the stored range is absolute against the underlying file. |
|
|
70
67
|
|
|
71
68
|
All methods may return synchronously or as a `Promise`. Implementations choose storage layout; callers should only rely on the returned `Binary` API.
|
|
72
69
|
|
|
@@ -81,12 +78,14 @@ export abstract class Binary {
|
|
|
81
78
|
```
|
|
82
79
|
|
|
83
80
|
The package activities return this `Binary` to your workflow — e.g. an `http`
|
|
84
|
-
response `Body` is backed by it.
|
|
85
|
-
|
|
81
|
+
response `Body` is backed by it. The workflow only passes the `Binary` to
|
|
82
|
+
another activity. An activity reads it in `run`:
|
|
86
83
|
|
|
87
84
|
```typescript
|
|
88
|
-
|
|
89
|
-
|
|
85
|
+
async run(file: Binary) {
|
|
86
|
+
const stream = await file.stream()
|
|
87
|
+
// consume stream (e.g. stream.toArray() then Buffer.concat)
|
|
88
|
+
}
|
|
90
89
|
```
|
|
91
90
|
|
|
92
91
|
---
|
|
@@ -95,7 +94,7 @@ const stream = await binary.stream()
|
|
|
95
94
|
|
|
96
95
|
Workflows never call `FileSystemClaim` directly. File I/O happens inside the
|
|
97
96
|
**ready-made package activities** that already own the claim — the `http` client
|
|
98
|
-
persists response bodies,
|
|
97
|
+
persists response bodies, the pdf render activity writes its output — and you
|
|
99
98
|
read the resulting `Binary` (e.g. `await res.body.json()` / `text()`). Your only
|
|
100
99
|
responsibility is that the runtime **registers a provider** (below). See
|
|
101
100
|
**Known consumers** for which package primitives to orchestrate.
|
|
@@ -120,7 +119,7 @@ The runtime must register a concrete subclass:
|
|
|
120
119
|
{ provide: FileSystemClaim, value: new MyFileSystemClaimImpl() }
|
|
121
120
|
```
|
|
122
121
|
|
|
123
|
-
`
|
|
122
|
+
`FileSystemClaim` is abstract, so `tsc` rejects a subclass that does not implement `write`, `read` and `slice`. The README uses `LocalFileSystemClaimImpl` as a **placeholder name** for that implementation; there is no class by that name in this monorepo.
|
|
124
123
|
|
|
125
124
|
---
|
|
126
125
|
|
|
@@ -131,8 +130,8 @@ These packages depend on `@t4h.framework/fs` and declare `FileSystemClaim`; read
|
|
|
131
130
|
| Package | Usage |
|
|
132
131
|
|---------|--------|
|
|
133
132
|
| `@t4h.framework/http` | The HTTP activity persists response bodies to `fs` and returns them wrapped in a `Body` — you read them with `res.body.json()` / `text()`. |
|
|
134
|
-
| `@t4h.framework/
|
|
135
|
-
| `@t4h.framework/fs` itself | `FileSplitActivity`
|
|
133
|
+
| `@t4h.framework/pdf` | The render activity writes the PDF to `fs` and hands back a `Binary` you consume in the workflow. |
|
|
134
|
+
| `@t4h.framework/fs` itself | `FileSplitActivity` streams the source and hands its slices to `startMany` as strings; `FileJoinActivity` streams batch results into a new file; `FileReadActivity` slices (or streams, when the binary is not file-backed) a small window into a string. See the `framework-fs-split` skill. |
|
|
136
135
|
|
|
137
136
|
Orchestrate these package primitives from your workflow; the `fs` write/read happens inside them.
|
|
138
137
|
|
|
@@ -145,8 +144,8 @@ Test the **workflow** with `TestWorkflowEnvironment` from
|
|
|
145
144
|
claims the triggered activities need, e.g. `HttpClientRequestClaim` for an HTTP
|
|
146
145
|
call). See the **framework-workflow-testing** skill for the harness API.
|
|
147
146
|
|
|
148
|
-
|
|
149
|
-
|
|
147
|
+
`@t4h.framework/core/no-binary-subclass` rejects a `Binary` subclass out of a
|
|
148
|
+
test file. A test double for the claim can extend `Binary`:
|
|
150
149
|
|
|
151
150
|
```typescript
|
|
152
151
|
import { Readable } from 'node:stream'
|
|
@@ -191,13 +190,12 @@ Assert `mockWrite` was called with the expected path and body when relevant.
|
|
|
191
190
|
|
|
192
191
|
## Implementing a custom FileSystemClaim
|
|
193
192
|
|
|
194
|
-
1. **Extend** `FileSystemClaim` from `@t4h.framework/fs`.
|
|
195
|
-
2. **Return** a `Binary`
|
|
196
|
-
3. **
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
5. **Keep paths deterministic** when activities replay — unpredictable temp paths are fine if only `Binary` metadata is serialized and replay re-executes `write`; know your serializer/replay story if `path` is persisted.
|
|
193
|
+
1. **Extend** `FileSystemClaim` from `@t4h.framework/fs`. `tsc` checks the method signatures.
|
|
194
|
+
2. **Return** a `Binary` from `write`, `read` and `slice` with correct `contentType`, `contentLength`, and `stream()`.
|
|
195
|
+
3. **Infer** the content type from the path when `write` gets no `contentType`.
|
|
196
|
+
4. **Compose** ranges in `slice`: a slice of a slice is relative to it.
|
|
197
|
+
5. **Register** `{ provide: FileSystemClaim, value: new YourImpl() }` in the runtime (tests use `mockClaim(FileSystemClaim, ...)` instead).
|
|
198
|
+
6. **Keep paths deterministic** when activities replay — unpredictable temp paths are fine if only `Binary` metadata is serialized and replay re-executes `write`; know your serializer/replay story if `path` is persisted.
|
|
201
199
|
|
|
202
200
|
---
|
|
203
201
|
|
|
@@ -206,14 +204,15 @@ Assert `mockWrite` was called with the expected path and body when relevant.
|
|
|
206
204
|
1. Add `"@t4h.framework/fs": "workspace:^"` (or published version) to app `dependencies` when activities use `FileSystemClaim`.
|
|
207
205
|
2. Ensure **`@t4h.framework/core`** is available (peer of `fs`).
|
|
208
206
|
3. Register **`FileSystemClaim`** in the runtime or test `claims` array.
|
|
209
|
-
4. Orchestrate file I/O through the package activities that own `FileSystemClaim` (http,
|
|
207
|
+
4. Orchestrate file I/O through the package activities that own `FileSystemClaim` (http, pdf) — workflow code never declares an activity or accesses `this.claims.fs`.
|
|
210
208
|
|
|
211
209
|
---
|
|
212
210
|
|
|
213
211
|
## Anti-patterns
|
|
214
212
|
|
|
215
|
-
-
|
|
216
|
-
-
|
|
213
|
+
- A deep import. The package exports only its entry, so a deep import fails in `tsc`.
|
|
214
|
+
- Calling `stream()` in a workflow, or in an activity `constructor`, `toInput` or `toOutput`. The `no-binary-stream-outside-activity` rule of `@t4h.framework/core/eslint` reports it.
|
|
215
|
+
- A missing `FileSystemClaim` provider. The activity throws `ClaimProviderNotFoundException` when it reads the claim.
|
|
217
216
|
- Assuming `packages/fs` ships a concrete disk/S3 implementation — it does not; implement and register `FileSystemClaim` in your runtime or test harness.
|
|
218
217
|
|
|
219
218
|
---
|
|
@@ -224,4 +223,4 @@ Assert `mockWrite` was called with the expected path and body when relevant.
|
|
|
224
223
|
- `framework-fs-write` — `write`: turn workflow content into a `Binary`, and why an App must never implement `Binary`.
|
|
225
224
|
- `framework-core` — `Claim`, `Binary`, and the `TestWorkflowEnvironment` testing harness.
|
|
226
225
|
- `framework-http` — HTTP activities that persist bodies via `FileSystemClaim`.
|
|
227
|
-
- `framework-
|
|
226
|
+
- `framework-pdf` — the render activity that writes PDFs via `fs`.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: framework-fs-split
|
|
3
3
|
description: >-
|
|
4
4
|
Guides correct use of `split`, `join` and `read` from @t4h.framework/fs:
|
|
5
|
-
fanning one large file out into a batch of workflows (one per
|
|
5
|
+
fanning one large file out into a batch of workflows (one per
|
|
6
6
|
slice), folding the batch results back into a single file, and reading small
|
|
7
7
|
byte windows (headers, trailers) into the workflow — with O(1) parent
|
|
8
8
|
history and bounded memory regardless of file size. Use whenever a workflow
|
|
@@ -20,8 +20,7 @@ description: >-
|
|
|
20
20
|
# `split` / `join` / `read` — file fan-out, fold and windows for workflows
|
|
21
21
|
|
|
22
22
|
`split` turns **one file into a batch of workflows**: it streams the file once,
|
|
23
|
-
cuts it into
|
|
24
|
-
child workflow per slice, and **suspends the parent until every child is
|
|
23
|
+
cuts it into slices, starts one child workflow per slice, and **suspends the parent until every child is
|
|
25
24
|
terminal**. `join` is the inverse: it folds the batch results back into a
|
|
26
25
|
single file, streaming one result at a time.
|
|
27
26
|
|
|
@@ -36,7 +35,7 @@ const processRecord = new Workflow(
|
|
|
36
35
|
},
|
|
37
36
|
)
|
|
38
37
|
|
|
39
|
-
const main = new Workflow(fileSchema, async (file: Binary) => {
|
|
38
|
+
const main = new Workflow({ id: 'main', schema: fileSchema }, async (file: Binary) => {
|
|
40
39
|
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
41
40
|
// header: string — recorded in history, never re-read on replay
|
|
42
41
|
|
|
@@ -62,13 +61,12 @@ Package path: `framework/packages/fs`. Import only from the package root.
|
|
|
62
61
|
Everything here exists to keep the parent history **O(1)** whatever the file
|
|
63
62
|
size:
|
|
64
63
|
|
|
65
|
-
1. `split` reads the source
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
`Type.String()`.
|
|
64
|
+
1. `split` reads the source once, worker side: it cuts records, drops the ones
|
|
65
|
+
`where` rejects, groups the rest by `every` and hands each slice to the
|
|
66
|
+
batch as a decoded string as soon as it is complete. Nothing collects the
|
|
67
|
+
slices, and no copy of the file is written.
|
|
68
|
+
2. Each child receives its slice as a plain string — what its
|
|
69
|
+
`Type.String()` schema declares.
|
|
72
70
|
3. The parent resumes with only `WorkflowBatchRef { id, count }`. Per-child
|
|
73
71
|
results are **never aggregated onto the ref**: they live with each child
|
|
74
72
|
run.
|
|
@@ -128,17 +126,16 @@ the next slice. Forgetting `{ next: 2 }` silently strips the record type code
|
|
|
128
126
|
from every slice.
|
|
129
127
|
|
|
130
128
|
Offsets are always computed over **raw bytes** (never decoded characters), so
|
|
131
|
-
multi-byte encodings cannot skew the
|
|
129
|
+
multi-byte encodings cannot skew the cuts.
|
|
132
130
|
|
|
133
131
|
### Other options
|
|
134
132
|
|
|
135
133
|
| Option | Default | Behavior |
|
|
136
134
|
|--------|---------|----------|
|
|
137
|
-
| `every` | `1` | Cut only every N delimiter occurrences — groups N records per child. One million lines with `every: 10_000` start 100 children, each receiving a raw block (inner delimiters untouched) to iterate locally. `keep` arithmetic applies only at actual cut points. |
|
|
135
|
+
| `every` | `1` | Cut only every N delimiter occurrences — groups N records per child. One million lines with `every: 10_000` start 100 children, each receiving a raw block (inner delimiters untouched) to iterate locally. `keep` arithmetic applies only at actual cut points. A slice over 3 MiB makes the split throw a `RangeError`; lower `every`. |
|
|
138
136
|
| `trim` | none | `{ start?, end? }` **bytes** cut off each end of the file before scanning — a positional header/trailer. Byte-based on purpose: a record-start delimiter (`'\n10'`) fuses the trailer into the last record's slice, so a record-count skip would drop a valid record along with it. Trimmed content never becomes a slice nor counts toward `every`; fetch it with `read`. Positional layouts make the byte counts constants — getting them right is the caller's business. |
|
|
139
137
|
| `skipEmpty` | `true` | Drops zero-length slices (consecutive delimiters, file starting/ending with one) so they never become child workflows. Turn off when slice index must align 1:1 with record position. "Empty" means zero bytes — a whitespace-only line is a regular slice. |
|
|
140
138
|
| `encoding` | `'utf8'` | Converts the delimiter to bytes and decodes each slice into the string the child receives. |
|
|
141
|
-
| `inlineMaxBytes` | `16_384` | A slice up to this many bytes is handed to the child as a string straight from the scan; a larger one as a reference to its byte range in the durable copy, read back where the child runs. Reading a few hundred bytes back costs a row lookup and a ranged object read per child, which is what a million-line split spends its CPU on. `0` hands every slice by range. Keep it well under the runtime's fan-out message budget (1 MiB by default). |
|
|
142
139
|
| `timeoutInMs` | none | Max milliseconds to await the batch; on expiry the activity **rejects** (reason `timeout`) instead of suspending forever — the safety net for a child that never reaches a terminal state. |
|
|
143
140
|
|
|
144
141
|
---
|
|
@@ -185,16 +182,12 @@ const relevant = await join<InvoiceResult>(batch, {
|
|
|
185
182
|
})
|
|
186
183
|
```
|
|
187
184
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
matches `['a', 'b']`), object values compare deeply, no type coercion. The
|
|
195
|
-
query compiles once and runs inside the existing streaming loop over
|
|
196
|
-
already-hydrated outputs — same memory profile; app-side (decides what enters
|
|
197
|
-
the file, not what leaves the database).
|
|
185
|
+
For an arbitrary predicate, write a custom activity in project code, so the
|
|
186
|
+
function lives in versioned code. MongoDB semantics apply: array fields match
|
|
187
|
+
by containment (`{ tags: 'a' }` matches `['a', 'b']`), object values compare
|
|
188
|
+
deeply, no type coercion. The query compiles once and runs inside the existing
|
|
189
|
+
streaming loop over already-hydrated outputs — same memory profile; app-side
|
|
190
|
+
(decides what enters the file, not what leaves the database).
|
|
198
191
|
|
|
199
192
|
---
|
|
200
193
|
|
|
@@ -208,11 +201,9 @@ async function read(
|
|
|
208
201
|
```
|
|
209
202
|
|
|
210
203
|
Reads a byte window of a `Binary` into the workflow as a decoded string. The
|
|
211
|
-
output is recorded in history (replay never re-reads the file)
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
defaults to the whole binary (same cap applies); `encoding` defaults to
|
|
215
|
-
`'utf8'`.
|
|
204
|
+
output is recorded in history (replay never re-reads the file). Use it for
|
|
205
|
+
headers, trailers and single records; a data file flows through `split`.
|
|
206
|
+
`range` defaults to the whole binary; `encoding` defaults to `'utf8'`.
|
|
216
207
|
|
|
217
208
|
Two paths under the hood:
|
|
218
209
|
|
|
@@ -248,8 +239,9 @@ const batch = await split(file, processInvoice, {
|
|
|
248
239
|
|
|
249
240
|
## Runtime requirements
|
|
250
241
|
|
|
251
|
-
- Claim providers: `FileSystemClaim` (with `slice`
|
|
252
|
-
`
|
|
242
|
+
- Claim providers: `FileSystemClaim` (with `slice`, for `read` and `join`),
|
|
243
|
+
`WorkflowClaim` (with `startMany` reading an `AsyncIterable`), and
|
|
244
|
+
`WorkflowBatchClaim` (from `@t4h.framework/core`).
|
|
253
245
|
- Production whitelist: `FileSplitActivity`, `FileJoinActivity`,
|
|
254
246
|
`FileReadActivity` and core's `StartManyWorkflowsActivity` must be
|
|
255
247
|
registered via `runtime.addActivity`.
|
|
@@ -271,26 +263,22 @@ from `src/activities/__tests__/`):
|
|
|
271
263
|
|
|
272
264
|
```typescript
|
|
273
265
|
import { AsyncActivityResultPending, Claim, WorkflowBatchClaim, WorkflowBatchRef, WorkflowClaim } from '@t4h.framework/core'
|
|
274
|
-
import { FileSplitActivity,
|
|
266
|
+
import { FileSplitActivity, Split } from '@t4h.framework/fs'
|
|
275
267
|
|
|
276
|
-
const
|
|
277
|
-
const
|
|
278
|
-
const
|
|
268
|
+
const inputs: unknown[] = []
|
|
269
|
+
const startMany = vi.fn(async (_wf, slices: AsyncIterable<unknown>) => {
|
|
270
|
+
for await (const slice of slices) inputs.push(slice) // must consume the stream
|
|
271
|
+
return new WorkflowBatchRef('batch-1', inputs.length)
|
|
272
|
+
})
|
|
279
273
|
|
|
280
274
|
const result = await Claim.run(
|
|
281
|
-
[
|
|
282
|
-
|
|
283
|
-
{
|
|
284
|
-
],
|
|
285
|
-
() => {
|
|
286
|
-
const activity = new FileSplitActivity()
|
|
287
|
-
activity.id = 'activity-1'
|
|
288
|
-
return activity.bootstrap({ binary, workflow: 'child', options: { delimiter: Split.line() } })
|
|
289
|
-
},
|
|
275
|
+
[{ provide: WorkflowClaim, value: { startMany } as never }],
|
|
276
|
+
() =>
|
|
277
|
+
new FileSplitActivity().bootstrap({ binary, workflow: 'child', options: { delimiter: Split.line() } }),
|
|
290
278
|
)
|
|
291
279
|
|
|
292
280
|
expect(result).toBeInstanceOf(AsyncActivityResultPending)
|
|
293
|
-
expect(
|
|
281
|
+
expect(inputs).toEqual(['foo', 'bar', 'baz'])
|
|
294
282
|
```
|
|
295
283
|
|
|
296
284
|
For `join`, provide `WorkflowBatchClaim` as an async generator:
|
|
@@ -301,9 +289,8 @@ const outputs = vi.fn(async function* () {
|
|
|
301
289
|
})
|
|
302
290
|
```
|
|
303
291
|
|
|
304
|
-
Behaviors worth asserting because they regress easily: the
|
|
305
|
-
|
|
306
|
-
`keep` arithmetic on positional cuts; the empty batch resolving immediately;
|
|
292
|
+
Behaviors worth asserting because they regress easily: the source is streamed
|
|
293
|
+
once; `keep` arithmetic on positional cuts; the empty batch resolving immediately;
|
|
307
294
|
CSV escaping (quotes doubled, delimiter/newline wrapped); the status filter.
|
|
308
295
|
|
|
309
296
|
---
|
|
@@ -330,12 +317,11 @@ CSV escaping (quotes doubled, delimiter/newline wrapped); the status filter.
|
|
|
330
317
|
- **Splitting a positional file without `trim`.** The header rides inside the
|
|
331
318
|
first slice (with `every: N` it silently consumes a record slot) and the
|
|
332
319
|
trailer fuses into the last one. Trim both ends; fetch them with `read`.
|
|
333
|
-
- **Reading a data file with `read`.** The output lands in workflow history
|
|
334
|
-
|
|
335
|
-
- **
|
|
336
|
-
|
|
337
|
-
`RegExp`
|
|
338
|
-
sift subset or a custom activity whose predicate lives in versioned code.
|
|
320
|
+
- **Reading a data file with `read`.** The output lands in workflow history.
|
|
321
|
+
Records flow through `split`. `read` throws a `RangeError` past 1 MiB.
|
|
322
|
+
- **A non-serializable `where`.** `tsc` rejects a function through
|
|
323
|
+
`WorkflowBatchWhere`. `toInput` throws a `TypeError` for a function,
|
|
324
|
+
`$where`, a `RegExp`, `undefined` or an operator outside the subset.
|
|
339
325
|
- **Hand-rolling the loop.** A stream-and-`startWorkflow` loop inside an
|
|
340
326
|
activity re-parses on replay, misses the barrier, and pushes N entries into
|
|
341
327
|
history.
|
|
@@ -37,7 +37,7 @@ Package path: `framework/packages/fs`. Import only from the package root.
|
|
|
37
37
|
|
|
38
38
|
## Never implement `Binary` yourself
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
`@t4h.framework/core/no-binary-subclass` rejects a class that extends `Binary`.
|
|
41
41
|
|
|
42
42
|
```typescript
|
|
43
43
|
// ❌ NEVER — this class cannot be persisted
|
|
@@ -89,8 +89,9 @@ Symptoms of doing it by hand:
|
|
|
89
89
|
- the activity is never persisted, the transaction rolls back, and the message
|
|
90
90
|
ends up in the dead letter queue after the retries — the run stops mid-way
|
|
91
91
|
with no obvious error in the workflow itself;
|
|
92
|
-
-
|
|
93
|
-
|
|
92
|
+
- the dev server also throws `TypeError: Unsupported binary instance` when it
|
|
93
|
+
stores the activity input. It accepts only the `Binary` that its own claims
|
|
94
|
+
return.
|
|
94
95
|
|
|
95
96
|
Current runtimes spool an unknown `Binary` to storage instead of failing, so a
|
|
96
97
|
hand-rolled class may survive — but it pays a hidden copy, loses the content
|
|
@@ -121,13 +122,13 @@ the recorded `Binary` is returned.
|
|
|
121
122
|
|
|
122
123
|
## When to use it
|
|
123
124
|
|
|
124
|
-
| Situation | Use
|
|
125
|
-
| ----------------------------------------------------------------------------- |
|
|
126
|
-
| Send content the workflow built (JSON, CSV, positional file) over SFTP/HTTP | `write`
|
|
127
|
-
| Read a small window of an existing file into the workflow | [`read`](./read) — see **framework-fs-split**
|
|
128
|
-
| Fan a large file out into one workflow per record | `split` — see **framework-fs-split**
|
|
129
|
-
| Fold a finished batch back into a file | `join` — see **framework-fs-split**
|
|
130
|
-
| Content already is a `Binary` (HTTP response body, `sftp.get`, `join` result) | pass it straight through
|
|
125
|
+
| Situation | Use |
|
|
126
|
+
| ----------------------------------------------------------------------------- | --------------------------------------------- |
|
|
127
|
+
| Send content the workflow built (JSON, CSV, positional file) over SFTP/HTTP | `write` |
|
|
128
|
+
| Read a small window of an existing file into the workflow | [`read`](./read) — see **framework-fs-split** |
|
|
129
|
+
| Fan a large file out into one workflow per record | `split` — see **framework-fs-split** |
|
|
130
|
+
| Fold a finished batch back into a file | `join` — see **framework-fs-split** |
|
|
131
|
+
| Content already is a `Binary` (HTTP response body, `sftp.get`, `join` result) | pass it straight through |
|
|
131
132
|
|
|
132
133
|
---
|
|
133
134
|
|
|
@@ -219,14 +220,11 @@ expect(write).toHaveBeenCalledWith('report.csv', expect.any(Buffer), 'text/csv')
|
|
|
219
220
|
|
|
220
221
|
## Anti-patterns
|
|
221
222
|
|
|
222
|
-
- **Declaring a class that extends `Binary`** in an App. There is no valid
|
|
223
|
-
reason to; use `write`.
|
|
224
223
|
- Helper functions like `stringToBinary(value)` that wrap content in a custom
|
|
225
224
|
class — replace the implementation with `write`, keeping the helper name if
|
|
226
225
|
it reads well.
|
|
227
|
-
- `write`-ing a value that already is a `Binary`
|
|
228
|
-
`
|
|
229
|
-
reference you already had.
|
|
226
|
+
- `write`-ing a value that already is a `Binary`. `write` takes only
|
|
227
|
+
`string | Buffer`, so `tsc` rejects it.
|
|
230
228
|
- Building a large file by concatenating records in memory and calling `write`
|
|
231
229
|
once — use `split`/`join` instead.
|
|
232
230
|
- Relying on the filename extension to set the content type when the consumer
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,49 @@
|
|
|
1
1
|
# @t4h.framework/fs
|
|
2
2
|
|
|
3
|
+
## 0.13.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#229](https://github.com/tech4humans-brasil/framework/pull/229) [`d4d7d97`](https://github.com/tech4humans-brasil/framework/commit/d4d7d973233ec1f4d78628a69283701b9a55fbf9) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - Fixes the published packages:
|
|
8
|
+
- harness: the published types import `typings/PublicProperties.js`, which was not in `dist`. `uuidv7` is now a dependency.
|
|
9
|
+
- dev: `tar` is now a dependency.
|
|
10
|
+
- fs: the published types of `FileSplitActivity` no longer import `typebox`, which `fs` does not depend on.
|
|
11
|
+
|
|
12
|
+
- [#241](https://github.com/tech4humans-brasil/framework/pull/241) [`f6b8454`](https://github.com/tech4humans-brasil/framework/commit/f6b845484d22e7c8e7240fe2e113f1cdea7c8ce3) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - Adds seven rules to `@t4h.framework/core/eslint` and turns them on in `configs.recommended`:
|
|
13
|
+
- `no-io-in-workflow` reports `fetch` and calls to network, file, database and mail modules in a workflow and in the `constructor`, `toInput` and `toOutput` of an activity. It also reports `zlib` and `stream` calls there, because they work on the content in memory on each replay.
|
|
14
|
+
- `no-binary-subclass` reports a class that extends `Binary`.
|
|
15
|
+
- `no-framework-activity-in-app` reports `new` and `extends` on an activity of a framework package other than the core.
|
|
16
|
+
- `no-testing-import-in-source` reports an import of a `@t4h.framework/<package>/testing` entry out of a test file.
|
|
17
|
+
- `require-workflow-id` reports a `Workflow` with no `id`.
|
|
18
|
+
- `no-secret-to-string` reports a `Secret` that becomes a string.
|
|
19
|
+
- `no-literal-secret-in-authorization` reports a string literal in the `secret` or `clientSecret` field of an `Authorization`.
|
|
20
|
+
|
|
21
|
+
The package skills now name these rules in place of the text that the rules enforce. The docs and skills give each `Workflow` an `id`, read the HMAC secret from `EnvDesign`, and do not document a custom HTTP activity.
|
|
22
|
+
|
|
23
|
+
- [#238](https://github.com/tech4humans-brasil/framework/pull/238) [`8567806`](https://github.com/tech4humans-brasil/framework/commit/8567806ec537288076c3a18a4816f26d58a67905) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - The `framework-fs` skill says to call `Binary.stream()` only in a method of an activity, not in the workflow.
|
|
24
|
+
|
|
25
|
+
- [#240](https://github.com/tech4humans-brasil/framework/pull/240) [`0606115`](https://github.com/tech4humans-brasil/framework/commit/0606115e2e0baa994e9f0ed3559ee54ecac88e44) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - The skills do not repeat a rule that a lint rule, `tsc` or a runtime error already enforces. Each such rule is now one line that names the gate. The examples that broke a gate (`process.env`, `as any`, an inline secret, an `EnvDesign` in a workflow file, `ref.output` before `wait()`) are correct now.
|
|
26
|
+
|
|
27
|
+
- [#228](https://github.com/tech4humans-brasil/framework/pull/228) [`d29e1ab`](https://github.com/tech4humans-brasil/framework/commit/d29e1ab29a039b13c3e4b075472bd615566ae75e) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - Uses exact types and `unknown` in place of `any`, and checks external data with guards.
|
|
28
|
+
- harness: the fix-issues prompt lists each sensor as `- checkTypes: PASS`. Before, it printed `- 0: [object Object]`.
|
|
29
|
+
- hmac: an algorithm name such as `constructor` throws `Unsupported algorithm`.
|
|
30
|
+
- oidc: a discovery response with no `introspection_endpoint`, or a null introspection body, throws a clear error.
|
|
31
|
+
- `HttpRequestActivity.toOutput` takes `HttpRequestActivityOutput` in place of `any`.
|
|
32
|
+
|
|
33
|
+
## 0.13.0
|
|
34
|
+
|
|
35
|
+
### Minor Changes
|
|
36
|
+
|
|
37
|
+
- [#214](https://github.com/tech4humans-brasil/framework/pull/214) [`fc4db5f`](https://github.com/tech4humans-brasil/framework/commit/fc4db5fa01454a45a3727b76dbacc215aa6b3ab7) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - `split` reads the file once, as a stream: it cuts the records, drops the ones
|
|
38
|
+
`where` rejects, groups the rest by `every` and hands each slice to
|
|
39
|
+
`startMany` as a string as soon as it is complete. No durable copy of the file
|
|
40
|
+
is written and no slice is read back by range. A slice may take up to 3 MiB
|
|
41
|
+
(the same as the runtime's default limit for the input of a workflow
|
|
42
|
+
run); past that the split fails while it reads, naming the byte the slice
|
|
43
|
+
starts at, where `where` used to cap a record at 1 MiB and nothing capped a
|
|
44
|
+
slice. `inlineMaxBytes` is removed: every slice reaches its child as a
|
|
45
|
+
string. `ByteDelimiterScanner` is removed from the public API.
|
|
46
|
+
|
|
3
47
|
## 0.12.2
|
|
4
48
|
|
|
5
49
|
## 0.12.1
|
package/README.md
CHANGED
|
@@ -145,7 +145,7 @@ which stream through storage.
|
|
|
145
145
|
|
|
146
146
|
### split
|
|
147
147
|
|
|
148
|
-
Splits a file into
|
|
148
|
+
Splits a file into slices, starts one child workflow per slice as
|
|
149
149
|
a single batch, and suspends the parent until every child is terminal —
|
|
150
150
|
regardless of each child succeeding or failing (a rejected child never
|
|
151
151
|
rejects the batch):
|
|
@@ -160,7 +160,7 @@ const processRecord = new Workflow(
|
|
|
160
160
|
},
|
|
161
161
|
)
|
|
162
162
|
|
|
163
|
-
const main = new Workflow(fileSchema, async (file: Binary) => {
|
|
163
|
+
const main = new Workflow({ id: 'main', schema: fileSchema }, async (file: Binary) => {
|
|
164
164
|
const batch = await split(file, processRecord, {
|
|
165
165
|
delimiter: Split.line(),
|
|
166
166
|
})
|
|
@@ -170,14 +170,15 @@ const main = new Workflow(fileSchema, async (file: Binary) => {
|
|
|
170
170
|
})
|
|
171
171
|
```
|
|
172
172
|
|
|
173
|
-
The
|
|
173
|
+
The source is read once, as a stream:
|
|
174
174
|
|
|
175
|
-
1. The source
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
175
|
+
1. The source is read once, worker side: cut into records, filtered by
|
|
176
|
+
`where` when given, grouped by `every`, and each slice goes into the batch
|
|
177
|
+
as a string as soon as it is complete. Nothing collects the slices, and a
|
|
178
|
+
slice may take up to 3 MiB: past that the split fails
|
|
179
|
+
while it reads, at the byte the slice starts.
|
|
180
|
+
2. The runtime reads that stream once and stores the batch as it goes; no copy
|
|
181
|
+
of the file is written and no slice is read back by range.
|
|
181
182
|
3. The activity output is just the batch reference (`{ id, count }`), so the
|
|
182
183
|
parent history stays O(1) whatever the slice count. Per-child results are
|
|
183
184
|
never aggregated: they live with each child run, to be inspected
|
|
@@ -222,8 +223,7 @@ await split(file, workflow, {
|
|
|
222
223
|
every: 10_000, // cut only every N occurrences (groups records)
|
|
223
224
|
trim: { start: 241, end: 241 }, // bytes cut off each end before scanning
|
|
224
225
|
skipEmpty: false, // keep zero-length slices (default: skipped)
|
|
225
|
-
encoding: 'latin1', // delimiter bytes + slice
|
|
226
|
-
inlineMaxBytes: 16_384, // slices up to this size travel as strings (default: 16 KiB)
|
|
226
|
+
encoding: 'latin1', // delimiter bytes + slice decoding (default: utf8)
|
|
227
227
|
timeoutInMs: 3_600_000, // max ms to await the batch; rejects on expiry
|
|
228
228
|
})
|
|
229
229
|
```
|
|
@@ -254,18 +254,9 @@ await split(file, workflow, {
|
|
|
254
254
|
workflows. Turn it off when the slice index must align 1:1 with the record
|
|
255
255
|
position in the source file. "Empty" means zero bytes: a whitespace-only
|
|
256
256
|
line is a regular slice.
|
|
257
|
-
- **`encoding`** converts the delimiter to bytes and
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
- **`inlineMaxBytes`** (default 16 KiB) is the size up to which a slice is
|
|
261
|
-
handed to the child as a string, straight from the scan. A slice above it is
|
|
262
|
-
handed as a reference to its byte range in the durable copy, read back
|
|
263
|
-
where the child runs. A line of a CSV is a few hundred bytes: reading it
|
|
264
|
-
back would cost a row lookup and a ranged object read per child, and that,
|
|
265
|
-
not the scan, is what a million-line split spends its CPU on. Set it to `0`
|
|
266
|
-
to hand every slice by range. Keep it well under the fan-out message budget
|
|
267
|
-
of the runtime (`WORKFLOW_BATCH_FANOUT_CHUNK_MAX_BYTES`, 1 MiB by default):
|
|
268
|
-
an inline slice that does not fit a message is refused.
|
|
257
|
+
- **`encoding`** converts the delimiter to bytes and decodes each slice into
|
|
258
|
+
the string its child receives. Offsets are always computed over raw bytes,
|
|
259
|
+
so multi-byte encodings never skew the cuts.
|
|
269
260
|
- **`timeoutInMs`** is the maximum time in milliseconds to await the batch. On
|
|
270
261
|
expiry the activity rejects (reason `timeout`) instead of suspending
|
|
271
262
|
forever — the safety net for a child that never reaches a terminal state.
|
|
@@ -374,21 +365,3 @@ already hydrated one at a time — the query compiles once and adds one
|
|
|
374
365
|
predicate call per entry, no extra serialization and no change to the memory
|
|
375
366
|
profile. It is applied app-side: every child output is still fetched and
|
|
376
367
|
parsed; `where` decides what enters the file, not what leaves the database.
|
|
377
|
-
|
|
378
|
-
### ByteDelimiterScanner
|
|
379
|
-
|
|
380
|
-
The scanner behind `split` is exported for direct use. It is a pass-through
|
|
381
|
-
`Transform`: chunks flow through unchanged (so the same pass can persist the
|
|
382
|
-
file) while it records slice offsets in absolute byte positions, handling
|
|
383
|
-
delimiters split across chunk boundaries. `ranges` is complete once the stream
|
|
384
|
-
has flushed.
|
|
385
|
-
|
|
386
|
-
```typescript
|
|
387
|
-
import { ByteDelimiterScanner } from '@t4h.framework/fs'
|
|
388
|
-
|
|
389
|
-
const scanner = new ByteDelimiterScanner({ delimiter: { value: '\n' } })
|
|
390
|
-
|
|
391
|
-
await pipeline(source, scanner, destination)
|
|
392
|
-
|
|
393
|
-
scanner.slices // [{ start: 0, end: 3, content: <Buffer> }, ...]
|
|
394
|
-
```
|
|
@@ -26,15 +26,15 @@ export type FileJoinActivityInput = {
|
|
|
26
26
|
options: FileJoinOptions;
|
|
27
27
|
filter?: WorkflowBatchFilter;
|
|
28
28
|
};
|
|
29
|
-
export declare class FileJoinActivity extends SyncActivity<readonly [batch: WorkflowBatchCollectionRef
|
|
29
|
+
export declare class FileJoinActivity extends SyncActivity<readonly [batch: WorkflowBatchCollectionRef, options: FileJoinOptions], FileJoinActivityInput, Binary, Binary> {
|
|
30
30
|
private readonly claims;
|
|
31
|
-
toInput(batch: WorkflowBatchCollectionRef
|
|
31
|
+
toInput(batch: WorkflowBatchCollectionRef, options: FileJoinOptions): FileJoinActivityInput;
|
|
32
32
|
run({ batch, options, filter, }: FileJoinActivityInput): Promise<Binary>;
|
|
33
33
|
toOutput(output: Binary): Binary;
|
|
34
34
|
private toStatus;
|
|
35
35
|
private writeHeader;
|
|
36
36
|
private toLine;
|
|
37
37
|
}
|
|
38
|
-
export declare function join<TOutput = unknown>(batch: WorkflowBatchCollectionRef
|
|
38
|
+
export declare function join<TOutput = unknown>(batch: WorkflowBatchCollectionRef, options: FileJoinOptions<TOutput>): Promise<Binary>;
|
|
39
39
|
export {};
|
|
40
40
|
//# sourceMappingURL=FileJoinActivity.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,YAAY,EAEZ,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EAGxB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAEvC,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAA;AAEhD,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE1E,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;IACxB,MAAM,CAAC,EAAE,mBAAmB,CAAA;CAC7B,CAAA;AAQD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,0BAA0B,
|
|
1
|
+
{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,YAAY,EAEZ,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EAGxB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAEvC,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAA;AAEhD,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE1E,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;IACxB,MAAM,CAAC,EAAE,mBAAmB,CAAA;CAC7B,CAAA;AAQD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,0BAA0B,EAAE,OAAO,EAAE,eAAe,CAAC,EACtE,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,0BAA0B,EACjC,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAkBX,GAAG,CAAC,EACf,KAAK,EACL,OAAO,EACP,MAAM,GACP,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IA6BnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAIvC,OAAO,CAAC,QAAQ;YAaF,WAAW;IAazB,OAAO,CAAC,MAAM;CAef;AAED,wBAAsB,IAAI,CAAC,OAAO,GAAG,OAAO,EAC1C,KAAK,EAAE,0BAA0B,EACjC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC,CAMjB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,EACP,YAAY,EACZ,kBAAkB,EAIlB,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAuCvC,MAAM,UAAU,GAA8C;IAC5D,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,QAAQ;IACjB,GAAG,EAAE,KAAK;CACX,CAAA;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,
|
|
1
|
+
{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,EACP,YAAY,EACZ,kBAAkB,EAIlB,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAuCvC,MAAM,UAAU,GAA8C;IAC5D,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,QAAQ;IACjB,GAAG,EAAE,KAAK;CACX,CAAA;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,KAAiC,EACjC,OAAwB;QAExB,IAAI,OAAO,CAAC,KAAK;YACf,kBAAkB,CAAC,kBAAkB,CACnC,OAAO,CAAC,KAAK,EACb,OAAO,EACP,YAAY,CACb,CAAA;QAEH,MAAM,MAAM,GAAwB;YAClC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAA;QAED,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;QAExE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;IAC7C,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EACf,KAAK,EACL,OAAO,EACP,MAAM,GACgB;QACtB,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAA;QAEhC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAClC,QAAQ,IAAI,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC/C,MAAM,CACP,CAAA;QAED,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAChC,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE;YAC/B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACtC,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CACH,CAAA;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAEtC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO;YAAE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;QAE3E,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,QAAQ,CACd,OAAwB,EACxB,MAA4B;QAE5B,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAA;QAEzC,IAAI,MAAM,EAAE,MAAM;YAAE,OAAM;QAE1B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;QAElE,OAAO,CAAC,WAAW,CAAC,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAwB,EACxB,KAAsC;QAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM;YAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAA;QAE3D,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YACtD,MAAM,KAAK,CACT,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAChE,CAAA;IACL,CAAC;IAEO,MAAM,CAAC,KAA0B,EAAE,OAAwB;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;QAErE,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YAC1B,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAA;QAEJ,MAAM,KAAK,GACT,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;YAC9B,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAElC,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAA;IAC5C,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,KAAiC,EACjC,OAAiC;IAEjC,OAAO,MAAM,OAAO,CAAC,UAAU,CAC7B,gBAAgB,EAChB,KAAK,EACL,OAA0B,CAC3B,CAAA;AACH,CAAC"}
|
|
@@ -17,6 +17,7 @@ export declare class FileReadActivity extends SyncActivity<readonly [binary: Bin
|
|
|
17
17
|
toOutput(output: string): string;
|
|
18
18
|
private slice;
|
|
19
19
|
private window;
|
|
20
|
+
private collect;
|
|
20
21
|
}
|
|
21
22
|
export declare function read(binary: Binary, options?: FileReadOptions): Promise<string>;
|
|
22
23
|
//# sourceMappingURL=FileReadActivity.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAID,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;
|
|
1
|
+
{"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAID,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;YAoBN,OAAO;CA8BtB;AAED,wBAAsB,IAAI,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAEjB"}
|