@t4h.framework/fs 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +28 -7
- package/.ai/skills/framework-fs-split/SKILL.md +113 -18
- package/.ai/skills/framework-fs-write/SKILL.md +242 -0
- package/CHANGELOG.md +78 -0
- package/LICENSE +21 -0
- package/README.md +154 -7
- package/dist/FileSystemClaim.d.ts +1 -1
- package/dist/FileSystemClaim.d.ts.map +1 -1
- package/dist/FileSystemClaim.js.map +1 -1
- package/dist/activities/FileJoinActivity.d.ts +32 -12
- package/dist/activities/FileJoinActivity.d.ts.map +1 -1
- package/dist/activities/FileJoinActivity.js +21 -34
- package/dist/activities/FileJoinActivity.js.map +1 -1
- package/dist/activities/FileReadActivity.d.ts +22 -0
- package/dist/activities/FileReadActivity.d.ts.map +1 -0
- package/dist/activities/FileReadActivity.js +68 -0
- package/dist/activities/FileReadActivity.js.map +1 -0
- package/dist/activities/FileSplitActivity.d.ts +2 -1
- package/dist/activities/FileSplitActivity.d.ts.map +1 -1
- package/dist/activities/FileSplitActivity.js +1 -0
- package/dist/activities/FileSplitActivity.js.map +1 -1
- package/dist/activities/FileWriteActivity.d.ts +20 -0
- package/dist/activities/FileWriteActivity.d.ts.map +1 -0
- package/dist/activities/FileWriteActivity.js +26 -0
- package/dist/activities/FileWriteActivity.js.map +1 -0
- package/dist/constants/operations.d.ts +3 -0
- package/dist/constants/operations.d.ts.map +1 -0
- package/dist/constants/operations.js +18 -0
- package/dist/constants/operations.js.map +1 -0
- package/dist/fs.d.ts +2 -0
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +2 -0
- package/dist/fs.js.map +1 -1
- package/dist/helpers/CSV.d.ts +14 -0
- package/dist/helpers/CSV.d.ts.map +1 -0
- package/dist/helpers/CSV.js +44 -0
- package/dist/helpers/CSV.js.map +1 -0
- package/dist/helpers/byte-delimiter-scanner.d.ts +8 -1
- package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -1
- package/dist/helpers/byte-delimiter-scanner.js +26 -2
- package/dist/helpers/byte-delimiter-scanner.js.map +1 -1
- package/dist/tools/assert-where-serializable.d.ts +2 -0
- package/dist/tools/assert-where-serializable.d.ts.map +1 -0
- package/dist/tools/assert-where-serializable.js +19 -0
- package/dist/tools/assert-where-serializable.js.map +1 -0
- package/package.json +6 -3
- package/dist/ActivityFileSystemClaim.d.ts +0 -7
- package/dist/ActivityFileSystemClaim.d.ts.map +0 -1
- package/dist/ActivityFileSystemClaim.js +0 -3
- package/dist/ActivityFileSystemClaim.js.map +0 -1
- package/dist/activities/__tests__/FileJoinActivity.spec.d.ts +0 -2
- package/dist/activities/__tests__/FileJoinActivity.spec.d.ts.map +0 -1
- package/dist/activities/__tests__/FileJoinActivity.spec.js +0 -160
- package/dist/activities/__tests__/FileJoinActivity.spec.js.map +0 -1
- package/dist/activities/__tests__/FileSplitActivity.spec.d.ts +0 -2
- package/dist/activities/__tests__/FileSplitActivity.spec.d.ts.map +0 -1
- package/dist/activities/__tests__/FileSplitActivity.spec.js +0 -149
- package/dist/activities/__tests__/FileSplitActivity.spec.js.map +0 -1
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts +0 -2
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts.map +0 -1
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +0 -127
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +0 -1
- package/dist/helpers/__tests__/csv-reader.spec.d.ts +0 -2
- package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +0 -1
- package/dist/helpers/__tests__/csv-reader.spec.js +0 -60
- package/dist/helpers/__tests__/csv-reader.spec.js.map +0 -1
- package/dist/helpers/csv-reader.d.ts +0 -8
- package/dist/helpers/csv-reader.d.ts.map +0 -1
- package/dist/helpers/csv-reader.js +0 -14
- package/dist/helpers/csv-reader.js.map +0 -1
- package/dist/helpers/parsers.d.ts +0 -9
- package/dist/helpers/parsers.d.ts.map +0 -1
- package/dist/helpers/parsers.js +0 -14
- package/dist/helpers/parsers.js.map +0 -1
- package/dist/testing/index.d.ts +0 -3
- package/dist/testing/index.d.ts.map +0 -1
- package/dist/testing/index.js +0 -32
- package/dist/testing/index.js.map +0 -1
|
@@ -4,6 +4,7 @@ description: >-
|
|
|
4
4
|
Guides correct use of @t4h.framework/fs in T4H Framework workflows: the
|
|
5
5
|
FileSystemClaim abstraction that the package activities (http, jsx) use to
|
|
6
6
|
read/write files, the Binary result workflows consume, and runtime providers.
|
|
7
|
+
A workflow never implements Binary itself — see the framework-fs-write skill.
|
|
7
8
|
Use when a workflow orchestrates file-backed package activities, consuming
|
|
8
9
|
Binary results, mocking FileSystemClaim in workflow tests, registering fs
|
|
9
10
|
claim providers, or debugging missing fs provider errors.
|
|
@@ -24,10 +25,13 @@ Public exports live in `src/fs.ts`:
|
|
|
24
25
|
```typescript
|
|
25
26
|
import {
|
|
26
27
|
FileSystemClaim,
|
|
27
|
-
// file fan-out / fold — see the framework-fs-split skill
|
|
28
|
+
// file fan-out / fold / windows — see the framework-fs-split skill
|
|
28
29
|
split, Split, FileSplitActivity, type FileSplitOptions,
|
|
29
|
-
join, FileJoinActivity, type FileJoinOptions,
|
|
30
|
-
|
|
30
|
+
join, FileJoinActivity, type FileJoinOptions, type FileJoinWhere,
|
|
31
|
+
read, FileReadActivity, type FileReadOptions,
|
|
32
|
+
// turn workflow content into a Binary — see the framework-fs-write skill
|
|
33
|
+
write, FileWriteActivity, type FileWriteOptions,
|
|
34
|
+
ByteDelimiterScanner, type FileSplitDelimiter, type ByteRange, type ByteTrim,
|
|
31
35
|
} from '@t4h.framework/fs'
|
|
32
36
|
```
|
|
33
37
|
|
|
@@ -45,6 +49,7 @@ export abstract class FileSystemClaim {
|
|
|
45
49
|
public abstract write(
|
|
46
50
|
path: string,
|
|
47
51
|
readable: Readable | Buffer,
|
|
52
|
+
contentType?: string,
|
|
48
53
|
): Binary | Promise<Binary>
|
|
49
54
|
|
|
50
55
|
public abstract read(path: string): Binary | Promise<Binary>
|
|
@@ -59,9 +64,9 @@ export abstract class FileSystemClaim {
|
|
|
59
64
|
|
|
60
65
|
| Method | Purpose |
|
|
61
66
|
|--------|---------|
|
|
62
|
-
| `write(path, readable)` | Persist stream or buffer at `path`; return a `Binary` handle (metadata + `stream()`). |
|
|
67
|
+
| `write(path, readable, contentType?)` | Persist stream or buffer at `path`; return a `Binary` handle (metadata + `stream()`). When `contentType` is omitted, infer it from the path. |
|
|
63
68
|
| `read(path)` | Load an existing file at `path`; return a `Binary` handle. |
|
|
64
|
-
| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `split` children receive plain strings. |
|
|
69
|
+
| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `split` children receive plain strings. Implementations must **compose ranges**: slicing a slice is relative to it, and the stored range is absolute against the underlying file. |
|
|
65
70
|
|
|
66
71
|
All methods may return synchronously or as a `Promise`. Implementations choose storage layout; callers should only rely on the returned `Binary` API.
|
|
67
72
|
|
|
@@ -95,6 +100,16 @@ read the resulting `Binary` (e.g. `await res.body.json()` / `text()`). Your only
|
|
|
95
100
|
responsibility is that the runtime **registers a provider** (below). See
|
|
96
101
|
**Known consumers** for which package primitives to orchestrate.
|
|
97
102
|
|
|
103
|
+
This package's own workflow-facing primitives are `split`, `join`, `read`
|
|
104
|
+
(small byte windows — headers, trailers — hydrated into the workflow as a
|
|
105
|
+
string, capped at 1 MiB) and `write` (content produced in the workflow turned
|
|
106
|
+
into a file). See the `framework-fs-split` and `framework-fs-write` skills.
|
|
107
|
+
|
|
108
|
+
**A workflow never implements `Binary`.** Every `Binary` comes from the
|
|
109
|
+
framework — an HTTP response body, `sftp.get`, `write`, `join`. A class holding
|
|
110
|
+
bytes in memory cannot be persisted, and every primitive it is handed to
|
|
111
|
+
persists its input. See the `framework-fs-write` skill.
|
|
112
|
+
|
|
98
113
|
---
|
|
99
114
|
|
|
100
115
|
## Runtime provider
|
|
@@ -117,7 +132,7 @@ These packages depend on `@t4h.framework/fs` and declare `FileSystemClaim`; read
|
|
|
117
132
|
|---------|--------|
|
|
118
133
|
| `@t4h.framework/http` | The HTTP activity persists response bodies to `fs` and returns them wrapped in a `Body` — you read them with `res.body.json()` / `text()`. |
|
|
119
134
|
| `@t4h.framework/jsx` | Render activities write their output to `fs` and hand back a `Binary` you consume in the workflow. |
|
|
120
|
-
| `@t4h.framework/fs` itself | `FileSplitActivity` persists the durable copy and slices it by ranges; `FileJoinActivity` streams batch results into a new file. See the `framework-fs-split` skill. |
|
|
135
|
+
| `@t4h.framework/fs` itself | `FileSplitActivity` persists the durable copy and slices it by ranges; `FileJoinActivity` streams batch results into a new file; `FileReadActivity` slices (or streams, when the binary is not file-backed) a small window into a string. See the `framework-fs-split` skill. |
|
|
121
136
|
|
|
122
137
|
Orchestrate these package primitives from your workflow; the `fs` write/read happens inside them.
|
|
123
138
|
|
|
@@ -130,6 +145,9 @@ Test the **workflow** with `TestWorkflowEnvironment` from
|
|
|
130
145
|
claims the triggered activities need, e.g. `HttpClientRequestClaim` for an HTTP
|
|
131
146
|
call). See the **framework-workflow-testing** skill for the harness API.
|
|
132
147
|
|
|
148
|
+
Subclassing `Binary` is legitimate **only here**, in a test double standing in
|
|
149
|
+
for the claim — never in workflow or App code.
|
|
150
|
+
|
|
133
151
|
```typescript
|
|
134
152
|
import { Readable } from 'node:stream'
|
|
135
153
|
import { Binary } from '@t4h.framework/core'
|
|
@@ -175,7 +193,9 @@ Assert `mockWrite` was called with the expected path and body when relevant.
|
|
|
175
193
|
|
|
176
194
|
1. **Extend** `FileSystemClaim` from `@t4h.framework/fs`.
|
|
177
195
|
2. **Return** a `Binary` subclass (or core-compatible `Binary`) from `write` and `read` with correct `contentType`, `contentLength`, and `stream()`.
|
|
178
|
-
3. **Accept** `Readable | Buffer` in `write`
|
|
196
|
+
3. **Accept** `Readable | Buffer` and the optional `contentType` in `write`
|
|
197
|
+
(match the abstract signature), falling back to inferring the content type
|
|
198
|
+
from the path when it is omitted.
|
|
179
199
|
4. **Register** `{ provide: FileSystemClaim, value: new YourImpl() }` in the runtime (tests use `mockClaim(FileSystemClaim, ...)` instead).
|
|
180
200
|
5. **Keep paths deterministic** when activities replay — unpredictable temp paths are fine if only `Binary` metadata is serialized and replay re-executes `write`; know your serializer/replay story if `path` is persisted.
|
|
181
201
|
|
|
@@ -201,6 +221,7 @@ Assert `mockWrite` was called with the expected path and body when relevant.
|
|
|
201
221
|
## Related skills
|
|
202
222
|
|
|
203
223
|
- `framework-fs-split` — `split`/`join`: fan a file out into a batch of workflows and fold the results back into a file.
|
|
224
|
+
- `framework-fs-write` — `write`: turn workflow content into a `Binary`, and why an App must never implement `Binary`.
|
|
204
225
|
- `framework-core` — `Claim`, `Binary`, and the `TestWorkflowEnvironment` testing harness.
|
|
205
226
|
- `framework-http` — HTTP activities that persist bodies via `FileSystemClaim`.
|
|
206
227
|
- `framework-jsx` — render activities that write/read via `fs`.
|
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: framework-fs-split
|
|
3
3
|
description: >-
|
|
4
|
-
Guides correct use of `split` and `
|
|
5
|
-
large file out into a batch of workflows (one per byte-range
|
|
6
|
-
folding the batch results back into a single file
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
4
|
+
Guides correct use of `split`, `join` and `read` from @t4h.framework/fs:
|
|
5
|
+
fanning one large file out into a batch of workflows (one per byte-range
|
|
6
|
+
slice), folding the batch results back into a single file, and reading small
|
|
7
|
+
byte windows (headers, trailers) into the workflow — with O(1) parent
|
|
8
|
+
history and bounded memory regardless of file size. Use whenever a workflow
|
|
9
|
+
must process a large file record by record — positional files with
|
|
10
|
+
header/trailer, plain-text lines, NDJSON, batch imports, per-record fan-out
|
|
11
|
+
with an aggregated output (e.g. a CSV report) — or when the user mentions
|
|
12
|
+
splitting/chunking a file, trimming a header/trailer, reading a file header,
|
|
13
|
+
filtering batch results, running a workflow per line/record, waiting for a
|
|
14
|
+
batch, collecting batch results, or the symbols `split`, `join`, `read`,
|
|
15
|
+
`Split`, `FileSplitActivity`, `FileJoinActivity`, `FileReadActivity`,
|
|
16
|
+
`WorkflowBatchRef`, `WorkflowBatchClaim`. Prefer this skill over
|
|
17
|
+
hand-rolling stream loops + `startWorkflow` calls.
|
|
15
18
|
---
|
|
16
19
|
|
|
17
|
-
# `split` / `join` — file fan-out and
|
|
20
|
+
# `split` / `join` / `read` — file fan-out, fold and windows for workflows
|
|
18
21
|
|
|
19
22
|
`split` turns **one file into a batch of workflows**: it streams the file once,
|
|
20
23
|
cuts it into **byte-range slices** (never copying slice content), starts one
|
|
@@ -23,7 +26,7 @@ terminal**. `join` is the inverse: it folds the batch results back into a
|
|
|
23
26
|
single file, streaming one result at a time.
|
|
24
27
|
|
|
25
28
|
```typescript
|
|
26
|
-
import { split, join, Split } from '@t4h.framework/fs'
|
|
29
|
+
import { read, split, join, Split } from '@t4h.framework/fs'
|
|
27
30
|
|
|
28
31
|
const processRecord = new Workflow(
|
|
29
32
|
{ id: 'process-record', schema: Type.String() },
|
|
@@ -34,9 +37,13 @@ const processRecord = new Workflow(
|
|
|
34
37
|
)
|
|
35
38
|
|
|
36
39
|
const main = new Workflow(fileSchema, async (file: Binary) => {
|
|
40
|
+
const header = await read(file, { range: { start: 0, end: 240 } })
|
|
41
|
+
// header: string — recorded in history, never re-read on replay
|
|
42
|
+
|
|
37
43
|
const batch = await split(file, processRecord, {
|
|
38
44
|
delimiter: Split.char('\n10', { next: 2 }),
|
|
39
|
-
|
|
45
|
+
trim: { start: 241, end: 241 }, // header/trailer stay out of the batch
|
|
46
|
+
timeoutInMs: 3_600_000,
|
|
40
47
|
})
|
|
41
48
|
// this line only runs once the WHOLE batch is terminal;
|
|
42
49
|
// batch: WorkflowBatchRef — { id: string, count: number }
|
|
@@ -128,9 +135,10 @@ multi-byte encodings cannot skew the ranges a ranged download will use.
|
|
|
128
135
|
| Option | Default | Behavior |
|
|
129
136
|
|--------|---------|----------|
|
|
130
137
|
| `every` | `1` | Cut only every N delimiter occurrences — groups N records per child. One million lines with `every: 10_000` start 100 children, each receiving a raw block (inner delimiters untouched) to iterate locally. `keep` arithmetic applies only at actual cut points. |
|
|
138
|
+
| `trim` | none | `{ start?, end? }` **bytes** cut off each end of the file before scanning — a positional header/trailer. Byte-based on purpose: a record-start delimiter (`'\n10'`) fuses the trailer into the last record's slice, so a record-count skip would drop a valid record along with it. Trimmed content never becomes a slice nor counts toward `every`; fetch it with `read`. Positional layouts make the byte counts constants — getting them right is the caller's business. |
|
|
131
139
|
| `skipEmpty` | `true` | Drops zero-length slices (consecutive delimiters, file starting/ending with one) so they never become child workflows. Turn off when slice index must align 1:1 with record position. "Empty" means zero bytes — a whitespace-only line is a regular slice. |
|
|
132
140
|
| `encoding` | `'utf8'` | Converts the delimiter to bytes and tells the runtime how to hydrate each slice for the child (string when set on the slice). |
|
|
133
|
-
| `
|
|
141
|
+
| `timeoutInMs` | none | Max milliseconds to await the batch; on expiry the activity **rejects** (reason `timeout`) instead of suspending forever — the safety net for a child that never reaches a terminal state. |
|
|
134
142
|
|
|
135
143
|
---
|
|
136
144
|
|
|
@@ -150,7 +158,7 @@ Formats (content is the **child's** job — the child returns the final line;
|
|
|
150
158
|
|--------|----------|
|
|
151
159
|
| `lines` | Concatenates raw child outputs, one per line. Options: `header`, `separator` (default `'\n'`). Non-string outputs are JSON-stringified. |
|
|
152
160
|
| `entries` | NDJSON dump of `{ index, status, output }`, one per line — the format-agnostic view of the batch, rejected children included by default. |
|
|
153
|
-
| `csv` | Projects object outputs into RFC 4180 escaped columns. Options: `columns` (required; entries are `{ key, header? }` — `key` selects the value on the child output, `header` labels the header row, plain string is shorthand for `{ key }`), `delimiter` (default `','`), `header` (default `true`). Missing keys become empty cells; nested values are JSON inside the cell. Value *formatting* belongs in the child workflow (sandboxed user code); the join only owns layout. |
|
|
161
|
+
| `csv` | Projects object outputs into RFC 4180 escaped columns. Options: `columns` (required; entries are `{ key, header? }` — `key` selects the value on the child output (a dot-path like `'meta.total'` reaches nested fields, matching the `where` semantics; a literal top-level key with dots wins over the nested path), `header` labels the header row, plain string is shorthand for `{ key }`), `delimiter` (default `','`), `header` (default `true`). Missing keys become empty cells; nested values selected whole are JSON inside the cell. Value *formatting* belongs in the child workflow (sandboxed user code); the join only owns layout. |
|
|
154
162
|
|
|
155
163
|
**Status filter** (`status?: ('fulfilled' | 'rejected')[]`) selects which
|
|
156
164
|
results make it into the file — there is no error policy. Defaults:
|
|
@@ -161,14 +169,92 @@ design; a reprocessing file of failures is one call away:
|
|
|
161
169
|
const failures = await join(batch, { format: 'entries', status: ['rejected'] })
|
|
162
170
|
```
|
|
163
171
|
|
|
172
|
+
**Where filter** (`where?`) narrows results with a **sift (MongoDB-style)
|
|
173
|
+
query** over each entry's output, applied after `status`. Whitelisted subset
|
|
174
|
+
only — declarative, JSON-only operators: `$eq`, `$ne`, `$lt`, `$lte`, `$gt`,
|
|
175
|
+
`$gte`, `$in`, `$nin`, `$exists`, `$not`, `$and`, `$or`, `$nor`, plus plain
|
|
176
|
+
equality and dot-paths for nested fields. Pass the child output type as a
|
|
177
|
+
generic to narrow top-level keys/values (dot-paths accepted, not narrowed):
|
|
178
|
+
|
|
179
|
+
```typescript
|
|
180
|
+
const relevant = await join<InvoiceResult>(batch, {
|
|
181
|
+
format: 'csv',
|
|
182
|
+
columns: ['id', 'total'],
|
|
183
|
+
where: { valid: true, total: { $gt: 100 }, 'meta.situacao': 'aprovada' },
|
|
184
|
+
})
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Non-serializable filters are rejected eagerly at `toInput` (`TypeError`):
|
|
188
|
+
functions (incl. `$where`), `RegExp` (incl. `$regex`), `undefined`, operators
|
|
189
|
+
outside the subset — a dropped closure would make the replay filter
|
|
190
|
+
differently than the first execution. Arbitrary predicates belong in a custom
|
|
191
|
+
activity in project code (the function then lives in versioned code).
|
|
192
|
+
MongoDB semantics apply: array fields match by containment (`{ tags: 'a' }`
|
|
193
|
+
matches `['a', 'b']`), object values compare deeply, no type coercion. The
|
|
194
|
+
query compiles once and runs inside the existing streaming loop over
|
|
195
|
+
already-hydrated outputs — same memory profile; app-side (decides what enters
|
|
196
|
+
the file, not what leaves the database).
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## `read`
|
|
201
|
+
|
|
202
|
+
```typescript
|
|
203
|
+
async function read(
|
|
204
|
+
binary: Binary,
|
|
205
|
+
options?: { range?: { start: number; end: number }; encoding?: BufferEncoding },
|
|
206
|
+
): Promise<string>
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Reads a byte window of a `Binary` into the workflow as a decoded string. The
|
|
210
|
+
output is recorded in history (replay never re-reads the file), so reads are
|
|
211
|
+
**capped at 1 MiB** — this is the primitive for headers, trailers and single
|
|
212
|
+
records; a data file flows through `split`, never through `read`. `range`
|
|
213
|
+
defaults to the whole binary (same cap applies); `encoding` defaults to
|
|
214
|
+
`'utf8'`.
|
|
215
|
+
|
|
216
|
+
Two paths under the hood:
|
|
217
|
+
|
|
218
|
+
1. **Fast path** — `FileSystemClaim.slice`: on a file-backed binary a trailer
|
|
219
|
+
read is a ranged download, whatever the file size.
|
|
220
|
+
2. **Streaming fallback** — binaries the claim cannot slice (e.g. an SFTP
|
|
221
|
+
reference): the stream is consumed up to the window and destroyed as soon
|
|
222
|
+
as it is filled. A header read pulls the first chunks only, whatever the
|
|
223
|
+
file size; a **trailing window over a non-sliceable binary** is the one
|
|
224
|
+
expensive case — the stream traverses the whole file to reach it.
|
|
225
|
+
|
|
226
|
+
Reading a slice is **relative to the slice** (`range: { start: 0, end: 10 }`
|
|
227
|
+
returns its first 10 bytes, not the underlying file's) — slice ranges compose.
|
|
228
|
+
|
|
229
|
+
The positional-file recipe — header/trailer in the parent, records in the
|
|
230
|
+
batch:
|
|
231
|
+
|
|
232
|
+
```typescript
|
|
233
|
+
const header = await read(file, { range: { start: 0, end: HEADER } })
|
|
234
|
+
const trailer = await read(file, {
|
|
235
|
+
range: { start: file.contentLength - TRAILER, end: file.contentLength },
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
const batch = await split(file, processInvoice, {
|
|
239
|
+
delimiter: Split.char('\n10', { next: 2 }),
|
|
240
|
+
trim: { start: HEADER + 1, end: TRAILER + 1 }, // +1: boundary newlines
|
|
241
|
+
})
|
|
242
|
+
|
|
243
|
+
// e.g. validate the trailer's record count against batch.count
|
|
244
|
+
```
|
|
245
|
+
|
|
164
246
|
---
|
|
165
247
|
|
|
166
248
|
## Runtime requirements
|
|
167
249
|
|
|
168
250
|
- Claim providers: `FileSystemClaim` (with `slice`), `WorkflowClaim` (with
|
|
169
251
|
`startMany`), and `WorkflowBatchClaim` (from `@t4h.framework/core`).
|
|
170
|
-
- Production whitelist: `FileSplitActivity`, `FileJoinActivity
|
|
171
|
-
`StartManyWorkflowsActivity` must be
|
|
252
|
+
- Production whitelist: `FileSplitActivity`, `FileJoinActivity`,
|
|
253
|
+
`FileReadActivity` and core's `StartManyWorkflowsActivity` must be
|
|
254
|
+
registered via `runtime.addActivity`.
|
|
255
|
+
- `FileSystemClaim.slice` implementations must **compose ranges** when slicing
|
|
256
|
+
an already-sliced binary (relative in, absolute against the underlying file
|
|
257
|
+
out) — `read` and slice consumers rely on that semantics.
|
|
172
258
|
- `FileSplitActivity` is an **AsyncActivity**: the heavy work happens in
|
|
173
259
|
`bootstrap`, executed worker side atomically with the pending suspension —
|
|
174
260
|
children cannot finish before the parent is awaiting them.
|
|
@@ -240,6 +326,15 @@ CSV escaping (quotes doubled, delimiter/newline wrapped); the status filter.
|
|
|
240
326
|
intact (hash it, upload it), pass the `Binary` through.
|
|
241
327
|
- **Forgetting `keep` on positional delimiters.** `Split.char('\n10')` without
|
|
242
328
|
`{ next: 2 }` silently discards the record type code from every slice.
|
|
329
|
+
- **Splitting a positional file without `trim`.** The header rides inside the
|
|
330
|
+
first slice (with `every: N` it silently consumes a record slot) and the
|
|
331
|
+
trailer fuses into the last one. Trim both ends; fetch them with `read`.
|
|
332
|
+
- **Reading a data file with `read`.** The output lands in workflow history;
|
|
333
|
+
the 1 MiB cap is a guard, not a target. Records flow through `split`.
|
|
334
|
+
- **Passing a function as a join filter.** Options serialize into history — a
|
|
335
|
+
closure cannot survive it, so `toInput` rejects functions, `$where`,
|
|
336
|
+
`RegExp` and out-of-subset operators with a `TypeError`. Use the `where`
|
|
337
|
+
sift subset or a custom activity whose predicate lives in versioned code.
|
|
243
338
|
- **Hand-rolling the loop.** A stream-and-`startWorkflow` loop inside an
|
|
244
339
|
activity re-parses on replay, misses the barrier, and pushes N entries into
|
|
245
340
|
history.
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: framework-fs-write
|
|
3
|
+
description: >-
|
|
4
|
+
Guides the use of `write` from @t4h.framework/fs — the only supported way to
|
|
5
|
+
turn content produced inside a workflow (a JSON string, a CSV, a positional
|
|
6
|
+
return file, a Buffer) into a `Binary` that other primitives accept. Use
|
|
7
|
+
whenever a workflow must upload, send or save content it built itself:
|
|
8
|
+
`sftp.put`/`sftp.append` with generated content, an HTTP request carrying a
|
|
9
|
+
file, persisting a report, or when the user mentions creating/building a
|
|
10
|
+
Binary, converting a string or Buffer to a Binary, "stringToBinary",
|
|
11
|
+
"BufferBinary", or hits `Unsupported binary instance`. A workflow must NEVER
|
|
12
|
+
implement its own `Binary` subclass — this skill explains why and what to do
|
|
13
|
+
instead.
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
# `write` — turn workflow content into a `Binary`
|
|
17
|
+
|
|
18
|
+
Every file primitive in the framework speaks `Binary`: `sftp.put`,
|
|
19
|
+
`sftp.append`, `split`, `join`, `pdf`, HTTP bodies. A `Binary` is a
|
|
20
|
+
**reference** to stored content, not the bytes themselves.
|
|
21
|
+
|
|
22
|
+
`write` is what produces one from content your workflow built:
|
|
23
|
+
|
|
24
|
+
```typescript
|
|
25
|
+
import { write } from '@t4h.framework/fs'
|
|
26
|
+
|
|
27
|
+
const file = await write(JSON.stringify(state, null, 2), {
|
|
28
|
+
contentType: 'application/json',
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
await sftp.put('/out/state.json', file)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Package path: `framework/packages/fs`. Import only from the package root.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## Never implement `Binary` yourself
|
|
39
|
+
|
|
40
|
+
This is the rule this skill exists to enforce:
|
|
41
|
+
|
|
42
|
+
```typescript
|
|
43
|
+
// ❌ NEVER — this class cannot be persisted
|
|
44
|
+
class BufferBinary extends Binary {
|
|
45
|
+
constructor(
|
|
46
|
+
private readonly content: Buffer,
|
|
47
|
+
readonly contentType = '...',
|
|
48
|
+
) {
|
|
49
|
+
super()
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
get contentLength() {
|
|
53
|
+
return this.content.length
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
stream() {
|
|
57
|
+
return Readable.from(this.content)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
await sftp.put('/out/state.json', new BufferBinary(Buffer.from(json)))
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```typescript
|
|
65
|
+
// ✅ ALWAYS
|
|
66
|
+
await sftp.put(
|
|
67
|
+
'/out/state.json',
|
|
68
|
+
await write(json, {
|
|
69
|
+
contentType: 'application/json',
|
|
70
|
+
}),
|
|
71
|
+
)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Why a custom `Binary` cannot work
|
|
75
|
+
|
|
76
|
+
A workflow is **durable**: the body replays, and each activity is persisted
|
|
77
|
+
before it runs. When you pass a value to a primitive, that value becomes the
|
|
78
|
+
activity **input**, and the input is serialized to storage.
|
|
79
|
+
|
|
80
|
+
A `Binary` holding bytes in memory has nothing to serialize into — the buffer
|
|
81
|
+
lives in the process that built it, and the activity may run in another
|
|
82
|
+
process, minutes later, after a replay. The runtime therefore has to spool the
|
|
83
|
+
content somewhere before the activity can be persisted at all.
|
|
84
|
+
|
|
85
|
+
Symptoms of doing it by hand:
|
|
86
|
+
|
|
87
|
+
- `TypeError: Unsupported binary instance` when the activity input is
|
|
88
|
+
serialized, on older runtimes;
|
|
89
|
+
- the activity is never persisted, the transaction rolls back, and the message
|
|
90
|
+
ends up in the dead letter queue after the retries — the run stops mid-way
|
|
91
|
+
with no obvious error in the workflow itself;
|
|
92
|
+
- it may still "work" locally, because the dev server keeps values in memory
|
|
93
|
+
and never crosses a process boundary. **Passing locally proves nothing here.**
|
|
94
|
+
|
|
95
|
+
Current runtimes spool an unknown `Binary` to storage instead of failing, so a
|
|
96
|
+
hand-rolled class may survive — but it pays a hidden copy, loses the content
|
|
97
|
+
type you meant to set, and depends on runtime-version behavior. `write` is the
|
|
98
|
+
contract.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## API
|
|
103
|
+
|
|
104
|
+
```typescript
|
|
105
|
+
function write(
|
|
106
|
+
content: string | Buffer,
|
|
107
|
+
options?: FileWriteOptions,
|
|
108
|
+
): Promise<Binary>
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
| Option | Type | Required | Purpose |
|
|
112
|
+
| ------------- | ---------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
113
|
+
| `contentType` | `string` | No | Content type stored with the file — `'application/json'`, `'text/csv'`, … Defaults to what the filename extension implies, then `application/octet-stream`. |
|
|
114
|
+
| `filename` | `string` | No | Name the file is stored under. Defaults to the activity id. |
|
|
115
|
+
| `encoding` | `BufferEncoding` | No | How a **string** `content` is encoded. Defaults to `'utf8'`. Ignored for a `Buffer`, which always travels as base64. |
|
|
116
|
+
|
|
117
|
+
`write` is a durable activity: on replay the content is **not** written again;
|
|
118
|
+
the recorded `Binary` is returned.
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## When to use it
|
|
123
|
+
|
|
124
|
+
| Situation | Use |
|
|
125
|
+
| ----------------------------------------------------------------------------- | ----------------------------------------------- |
|
|
126
|
+
| Send content the workflow built (JSON, CSV, positional file) over SFTP/HTTP | `write` |
|
|
127
|
+
| Read a small window of an existing file into the workflow | [`read`](./read) — see **framework-fs-split** |
|
|
128
|
+
| Fan a large file out into one workflow per record | `split` — see **framework-fs-split** |
|
|
129
|
+
| Fold a finished batch back into a file | `join` — see **framework-fs-split** |
|
|
130
|
+
| Content already is a `Binary` (HTTP response body, `sftp.get`, `join` result) | pass it straight through — do not re-`write` it |
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Size matters: keep content bounded
|
|
135
|
+
|
|
136
|
+
`write` takes content **in workflow memory**, so it is bounded by what the
|
|
137
|
+
workflow can reasonably hold — a state file, a report, a return file. It is not
|
|
138
|
+
the tool for gigabyte outputs.
|
|
139
|
+
|
|
140
|
+
To build a large file out of many records, use `join` over a batch: it streams
|
|
141
|
+
results one at a time into storage, and nothing but the resulting `Binary`
|
|
142
|
+
reaches the workflow.
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Patterns
|
|
147
|
+
|
|
148
|
+
### A) Write then upload
|
|
149
|
+
|
|
150
|
+
```typescript
|
|
151
|
+
const csv = await write(rows.map(toCsvLine).join('\n'), {
|
|
152
|
+
contentType: 'text/csv',
|
|
153
|
+
filename: `report-${input.date}.csv`,
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
await sftp.put(`/out/report-${input.date}.csv`, csv)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### B) Write once, use twice
|
|
160
|
+
|
|
161
|
+
The same `Binary` can be handed to more than one primitive — it is a reference,
|
|
162
|
+
so nothing is copied and nothing is re-uploaded:
|
|
163
|
+
|
|
164
|
+
```typescript
|
|
165
|
+
const file = await write(json, { contentType: 'application/json' })
|
|
166
|
+
|
|
167
|
+
await sftp.put(`${path}.tmp`, file)
|
|
168
|
+
await sftp.put(path, file)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Note that a hand-rolled `Binary` backed by a single-use stream would break on
|
|
172
|
+
the second call — another reason not to build one.
|
|
173
|
+
|
|
174
|
+
### C) Binary content
|
|
175
|
+
|
|
176
|
+
```typescript
|
|
177
|
+
const pdfBytes: Buffer = await buildPdf()
|
|
178
|
+
|
|
179
|
+
await sftp.put(
|
|
180
|
+
'/out/invoice.pdf',
|
|
181
|
+
await write(pdfBytes, {
|
|
182
|
+
contentType: 'application/pdf',
|
|
183
|
+
}),
|
|
184
|
+
)
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## Testing
|
|
190
|
+
|
|
191
|
+
`write` goes through `FileSystemClaim.write`, so mock that claim like any
|
|
192
|
+
other. See the **framework-workflow-testing** skill for the harness API.
|
|
193
|
+
|
|
194
|
+
```typescript
|
|
195
|
+
import { TestWorkflowEnvironment, mockClaim } from '@t4h.framework/core/testing'
|
|
196
|
+
import { FileSystemClaim } from '@t4h.framework/fs'
|
|
197
|
+
|
|
198
|
+
// replay runs the workflow twice — return a fresh Binary per call
|
|
199
|
+
const write = vi
|
|
200
|
+
.fn()
|
|
201
|
+
.mockImplementation(
|
|
202
|
+
async (path, content, contentType) =>
|
|
203
|
+
new MockBinary(
|
|
204
|
+
Buffer.isBuffer(content) ? content : await toBuffer(content),
|
|
205
|
+
contentType,
|
|
206
|
+
),
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
await env.execute(workflow, input, {
|
|
210
|
+
claims: [
|
|
211
|
+
mockClaim(FileSystemClaim, { write, read: vi.fn(), slice: vi.fn() }),
|
|
212
|
+
],
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
expect(write).toHaveBeenCalledWith('report.csv', expect.any(Buffer), 'text/csv')
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## Anti-patterns
|
|
221
|
+
|
|
222
|
+
- **Declaring a class that extends `Binary`** in an App. There is no valid
|
|
223
|
+
reason to; use `write`.
|
|
224
|
+
- Helper functions like `stringToBinary(value)` that wrap content in a custom
|
|
225
|
+
class — replace the implementation with `write`, keeping the helper name if
|
|
226
|
+
it reads well.
|
|
227
|
+
- `write`-ing a value that already is a `Binary` (an HTTP response body, a
|
|
228
|
+
`sftp.get` result, a `join` output) — that copies a file to obtain a
|
|
229
|
+
reference you already had.
|
|
230
|
+
- Building a large file by concatenating records in memory and calling `write`
|
|
231
|
+
once — use `split`/`join` instead.
|
|
232
|
+
- Relying on the filename extension to set the content type when the consumer
|
|
233
|
+
cares about it — pass `contentType` explicitly.
|
|
234
|
+
|
|
235
|
+
---
|
|
236
|
+
|
|
237
|
+
## Related skills
|
|
238
|
+
|
|
239
|
+
- `framework-fs` — the `FileSystemClaim` contract and runtime providers.
|
|
240
|
+
- `framework-fs-split` — `read`/`split`/`join`: windows, fan-out and fold.
|
|
241
|
+
- `framework-sftp` — `sftp.put`/`append`, the most common consumer of `write`.
|
|
242
|
+
- `framework-workflow-creation` — the rules a workflow body must follow.
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# @t4h.framework/fs
|
|
2
|
+
|
|
3
|
+
## 0.8.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- [`8c56f76`](https://github.com/tech4humans-brasil/framework/commit/8c56f7697390a1ea966d1e91ff5f35ce5751a0d7) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - feat: write in-memory content to durable storage from a workflow
|
|
8
|
+
|
|
9
|
+
There was no supported way to turn a string or a Buffer into a `Binary` inside
|
|
10
|
+
a workflow — every `Binary` came from an existing file (`http` response,
|
|
11
|
+
`sftp.get`, `split`, `join`). Anyone needing to upload computed content had to
|
|
12
|
+
implement `Binary` by hand, and such an instance can never be persisted, so the
|
|
13
|
+
activity carrying it failed to serialize.
|
|
14
|
+
|
|
15
|
+
`write(content, options?)` covers that gap: it pushes an activity that stores
|
|
16
|
+
the content through the `FileSystemClaim` and returns a `Binary` that survives
|
|
17
|
+
replay.
|
|
18
|
+
|
|
19
|
+
```ts
|
|
20
|
+
const file = await write(JSON.stringify(data), {
|
|
21
|
+
contentType: 'application/json',
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
await sftp.put('/out/data.json', file)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
`FileSystemClaim.write` takes an optional `contentType` — until now the content
|
|
28
|
+
type was always inferred from the file extension.
|
|
29
|
+
|
|
30
|
+
The dev codec no longer diverges from the runtime one: its binary kind used to
|
|
31
|
+
match only `DevFileBinary`, so a hand-written `Binary` was silently serialized
|
|
32
|
+
as a plain object locally and blew up once deployed. It now matches any
|
|
33
|
+
`Binary` and spools unknown ones to disk, the same way the runtime spools them
|
|
34
|
+
to the file storage.
|
|
35
|
+
|
|
36
|
+
- [`8c56f76`](https://github.com/tech4humans-brasil/framework/commit/8c56f7697390a1ea966d1e91ff5f35ce5751a0d7) Thanks [@gusteycamargo](https://github.com/gusteycamargo)! - Adds FS write activity
|
|
37
|
+
|
|
38
|
+
## 0.7.0
|
|
39
|
+
|
|
40
|
+
### Minor Changes
|
|
41
|
+
|
|
42
|
+
- Adds support to read file ([#109](https://github.com/tech4humans-brasil/framework/pull/109))
|
|
43
|
+
|
|
44
|
+
## 0.6.0
|
|
45
|
+
|
|
46
|
+
### Minor Changes
|
|
47
|
+
|
|
48
|
+
- Adds activity to split and join binary ([#100](https://github.com/tech4humans-brasil/framework/pull/100))
|
|
49
|
+
|
|
50
|
+
## 0.5.0
|
|
51
|
+
|
|
52
|
+
### Patch Changes
|
|
53
|
+
|
|
54
|
+
- Update skills ([`6d7e7bc`](https://github.com/tech4humans-brasil/framework/commit/6d7e7bca0c780a9c53a0c9e5aa0a865c5af67959))
|
|
55
|
+
|
|
56
|
+
## 0.4.0
|
|
57
|
+
|
|
58
|
+
### Patch Changes
|
|
59
|
+
|
|
60
|
+
- Internal dependency updates
|
|
61
|
+
|
|
62
|
+
## 0.3.0
|
|
63
|
+
|
|
64
|
+
### Minor Changes
|
|
65
|
+
|
|
66
|
+
- Adds fs package ([`538c9bd`](https://github.com/tech4humans-brasil/framework/commit/538c9bd5c49031ceb94ef8d6c513f6ffc9c848c5))
|
|
67
|
+
- Adds support to buffer in fs write package ([`476f9f0`](https://github.com/tech4humans-brasil/framework/commit/476f9f08a7b379a3361e42f8577078462276210b))
|
|
68
|
+
- Adds .ai/skills in packages ([#88](https://github.com/tech4humans-brasil/framework/pull/88))
|
|
69
|
+
|
|
70
|
+
### Patch Changes
|
|
71
|
+
|
|
72
|
+
- Remove activity prefix from file system claim class name ([`d257bd1`](https://github.com/tech4humans-brasil/framework/commit/d257bd177d477a56a897ba862a0be1f50531c633))
|
|
73
|
+
- Adds github workflow to run, build, tests and lint ([#18](https://github.com/tech4humans-brasil/framework/pull/18))
|
|
74
|
+
- Adds readme ([`3a55794`](https://github.com/tech4humans-brasil/framework/commit/3a557942769fa022b9a39504046d22b0083e734f))
|
|
75
|
+
- Update package metadata and author information across multiple packages ([`2ca73df`](https://github.com/tech4humans-brasil/framework/commit/2ca73df42dc63c86ed44717b50d5402fca45b4e7))
|
|
76
|
+
- Improve repository info ([`1fad92d`](https://github.com/tech4humans-brasil/framework/commit/1fad92d959b804189fd1486d047115995ac81eb0))
|
|
77
|
+
- Fix resolve module resolution issues in check-types ([`c97a0d2`](https://github.com/tech4humans-brasil/framework/commit/c97a0d2747f38e561cc5c7d592b20de0459bc1ee))
|
|
78
|
+
- Update package dependencies across multiple packages ([`8b4fc1d`](https://github.com/tech4humans-brasil/framework/commit/8b4fc1d0e92d6bd880ddf4c8119fb68c49c53760))
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Tech for humans
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|