@t4h.framework/fs 0.13.1 → 0.14.2

package/README.md

@@ -1,367 +1,51 @@
 # @t4h.framework/fs
 
-File system claim for reading and writing files within activities, plus the
-workflow-facing file activities: `split` fans a large file out into one child
-workflow per slice, `join` folds the batch results back into a single file,
-`read` brings a small byte window (a header, a trailer) into the workflow as a
-string, and `write` turns content the workflow produced into a file.
+File functions for T4H Framework workflows. `read` brings a small byte window
+into the workflow, `write` turns workflow content into a file, `split` starts
+one child workflow per record of a file, `join` folds the batch results back
+into one file, and `asset` reads a file that ships with the project. Only
+references to the files go into the workflow history.
 
-## Claim
+## Install
 
-### FileSystemClaim
-
-Abstract dependency for file system operations. An activity declares this claim when it needs to read or write files. The runtime provides a concrete implementation (e.g. local disk, S3, GCS).
-
-```typescript
-import { FileSystemClaim } from '@t4h.framework/fs'
-```
-
-### Interface
-
-```typescript
-abstract class FileSystemClaim {
-  public abstract write(
-    path: string,
-    readable: Readable | Buffer,
-    contentType?: string,
-  ): Binary | Promise<Binary>
-  public abstract read(path: string): Binary | Promise<Binary>
-
-  public abstract slice(
-    binary: Binary,
-    range: { start: number; end: number },
-    encoding?: BufferEncoding,
-  ): Binary | Promise<Binary>
-}
-```
-
-### Usage inside an activity
-
-```typescript
-import { SyncActivity, Claim } from '@t4h.framework/core'
-import { FileSystemClaim } from '@t4h.framework/fs'
-
-class SaveReportActivity extends SyncActivity<
-  [data: Buffer],
-  { path: string },
-  any
-> {
-  private readonly claims = new Claim({
-    fs: FileSystemClaim,
-  })
-
-  public async toInput(data: Buffer) {
-    return { path: 'reports/output.pdf' }
-  }
-
-  public async run(input: { path: string }) {
-    const binary = await this.claims.fs.write(input.path, Buffer.from('...'))
-    return { contentType: binary.contentType }
-  }
-
-  public toOutput(output: any) {
-    return output
-  }
-}
-```
-
-### Runtime provider
-
-The runtime must supply a concrete implementation:
-
-```typescript
-{ provide: FileSystemClaim, value: new LocalFileSystemClaimImpl() }
-```
-
-## File read
-
-### read
-
-Reads a byte window of a `Binary` into the workflow as a decoded string. The
-output is recorded in the workflow history (replay never re-reads the file),
-which is why reads are capped at 1 MiB: this is the primitive for headers,
-trailers and single records — a data file flows through `split`, never through
-`read`.
-
-```typescript
-import { read } from '@t4h.framework/fs'
-
-const header = await read(file, { range: { start: 0, end: 240 } })
-
-const trailer = await read(file, {
-  range: { start: file.contentLength - 240, end: file.contentLength },
-})
+```bash
+npm install @t4h.framework/fs @t4h.framework/core
 ```
 
-Options: `range` (default: the whole binary, subject to the same cap) and
-`encoding` (default `'utf8'`).
+Requires Node.js 22.13 or later. Peer dependency: `@t4h.framework/core`.
 
-The activity tries `FileSystemClaim.slice` first — on a file-backed binary a
-trailer read is a ranged download, whatever the file size. Binaries the claim
-cannot slice (e.g. an SFTP reference) fall back to streaming: the stream is
-consumed up to the window and destroyed as soon as it is filled, so a header
-read pulls the first chunks only, whatever the file size. The one expensive
-case is a trailing window over a non-sliceable binary — the stream must
-traverse the whole file to reach it.
-
-Reading a slice is relative to the slice: `read(slice, { range: { start: 0,
-end: 10 } })` returns the first 10 bytes of the slice, not of the underlying
-file — slice ranges compose.
-
-## File write
-
-### write
-
-Stores content produced inside the workflow and returns its `Binary` — the only
-supported way to obtain a `Binary` from a string or a `Buffer`:
+## Usage
 
 ```typescript
-import { write } from '@t4h.framework/fs'
-
-const file = await write(JSON.stringify(state, null, 2), {
-  contentType: 'application/json',
-})
+import Type from 'typebox'
 
-await sftp.put('/out/state.json', file)
-```
-
-Options: `contentType` (default: inferred from the filename, then
-`application/octet-stream`), `filename` (default: the activity id) and
-`encoding` for a string content (default `'utf8'`; a `Buffer` always travels as
-base64).
-
-**Never implement `Binary` in an App.** Passing a value to a primitive makes it
-the input of a durable activity, and that input is serialized to storage before
-the activity runs — a `Binary` holding bytes in memory has nothing to serialize
-into, and the activity may run in another process after a replay. A hand-rolled
-class can pass locally (the dev server keeps values in memory) and fail once
-deployed.
-
-`write` takes content already in workflow memory — a state file, a report, a
-return file. To build a large file out of many records, use `split` + `join`,
-which stream through storage.
-
-## File split
-
-### split
-
-Splits a file into slices, starts one child workflow per slice as
-a single batch, and suspends the parent until every child is terminal —
-regardless of each child succeeding or failing (a rejected child never
-rejects the batch):
-
-```typescript
-import { split, Split } from '@t4h.framework/fs'
+import { type Binary, Workflow } from '@t4h.framework/core'
+import { join, split, Split } from '@t4h.framework/fs'
 
 const processRecord = new Workflow(
   { id: 'process-record', schema: Type.String() },
-  async (record: string) => {
-    // record is the raw slice content
-  },
+  async (record: string) => record.slice(0, 10),
 )
 
-const main = new Workflow({ id: 'main', schema: fileSchema }, async (file: Binary) => {
-  const batch = await split(file, processRecord, {
-    delimiter: Split.line(),
-  })
+const main = new Workflow(
+  { id: 'main', schema: Type.Any() },
+  async (file: Binary) => {
+    const batch = await split(file, processRecord, {
+      delimiter: Split.line(),
+      timeoutInMs: 3_600_000,
+    })
 
-  // this line only runs once the whole batch is terminal;
-  // batch: WorkflowBatchRef — { id: string, count: number }
-})
-```
-
-The source is read once, as a stream:
-
-1. The source is read once, worker side: cut into records, filtered by
-   `where` when given, grouped by `every`, and each slice goes into the batch
-   as a string as soon as it is complete. Nothing collects the slices, and a
-   slice may take up to 3 MiB: past that the split fails
-   while it reads, at the byte the slice starts.
-2. The runtime reads that stream once and stores the batch as it goes; no copy
-   of the file is written and no slice is read back by range.
-3. The activity output is just the batch reference (`{ id, count }`), so the
-   parent history stays O(1) whatever the slice count. Per-child results are
-   never aggregated: they live with each child run, to be inspected
-   individually through the batch id.
-
-Child workflows declare their real input type (e.g. `Type.String()`); how the
-bytes reach them is a runtime concern.
-
-### Delimiters
-
-The cut is defined by the dev through a serializable delimiter spec. When the
-delimiter is found at `[p, p + length)`, the previous slice ends at
-`p + keep.previous` and the next slice starts at `p + length - keep.next`; the
-bytes in between are discarded. The defaults discard the whole delimiter
-(classic line semantics).
-
-```typescript
-Split.line() // cut at every '\n', delimiter discarded
-Split.char(';') // cut at any byte sequence
-Split.char(';', { previous: 1 }) // keep the ';' with the previous slice
-Split.char('\n10', { next: 2 }) // positional files, see below
-```
-
-The last form targets positional files where a logical record spans multiple
-physical lines and a new record only starts when a line begins with a marker
-(the record type code):
-
-```
-10JOHN;123\n20detail-a\n20detail-b\n10MARY;456\n20detail-c
-└──────────── record 1 ───────────┘└─────── record 2 ──────┘
-```
-
-With `Split.char('\n10', { next: 2 })` the newline between records is
-discarded, and the `10` marker — which is data, not separator — stays at the
-start of the next slice.
-
-### Options
-
-```typescript
-await split(file, workflow, {
-  delimiter: Split.line(), // required: where to cut
-  every: 10_000, // cut only every N occurrences (groups records)
-  trim: { start: 241, end: 241 }, // bytes cut off each end before scanning
-  skipEmpty: false, // keep zero-length slices (default: skipped)
-  encoding: 'latin1', // delimiter bytes + slice decoding (default: utf8)
-  timeoutInMs: 3_600_000, // max ms to await the batch; rejects on expiry
-})
-```
-
-- **`every`** groups N records per child. With one million lines, `every:
-10_000` starts 100 children instead of one million; each child receives a
-  raw block with its inner delimiters untouched and iterates locally. `keep`
-  arithmetic only applies at actual cut points.
-- **`trim`** cuts a fixed number of bytes off each end of the file before
-  scanning — a positional header/trailer, which may not be aligned with the
-  record delimiter (a record-start delimiter like `Split.char('\n10')` fuses
-  the trailer into the last record's slice, so counting records would drop a
-  valid one along with it). Trimmed content never becomes a slice nor counts
-  toward `every`; fetch it with `read`. Getting the byte counts right is the
-  caller's business — positional layouts make them constants:
-
-  ```typescript
-  const header = await read(file, { range: { start: 0, end: 240 } })
-
-  const batch = await split(file, processInvoice, {
-    delimiter: Split.char('\n10', { next: 2 }),
-    trim: { start: 241, end: 241 }, // header + '\n', trailer + '\n'
-  })
-  ```
-
-- **`skipEmpty`** (default `true`) drops zero-length slices — consecutive
-  delimiters, or a file starting/ending with one — so they never become child
-  workflows. Turn it off when the slice index must align 1:1 with the record
-  position in the source file. "Empty" means zero bytes: a whitespace-only
-  line is a regular slice.
-- **`encoding`** converts the delimiter to bytes and decodes each slice into
-  the string its child receives. Offsets are always computed over raw bytes,
-  so multi-byte encodings never skew the cuts.
-- **`timeoutInMs`** is the maximum time in milliseconds to await the batch. On
-  expiry the activity rejects (reason `timeout`) instead of suspending
-  forever — the safety net for a child that never reaches a terminal state.
-
-## File join
-
-The inverse of `split`: folds the results of a completed batch into a single
-file. Outputs stream in one at a time through `WorkflowBatchClaim` and flow
-straight into `FileSystemClaim.write` with backpressure, so memory stays
-bounded whatever the batch size — only the resulting binary reaches the
-workflow history.
-
-```typescript
-import { join, split, Split } from '@t4h.framework/fs'
-
-const batch = await split(file, processRecord, { delimiter: Split.line() })
-
-const csv = await join(batch, {
-  format: 'csv',
-  columns: [
-    { key: 'id', header: 'Código' },
-    { key: 'value', header: 'Valor (R$)' },
-    'date', // shorthand: key === header
-  ],
-})
-// csv: Binary
-```
-
-### Formats
-
-- **`lines`** — concatenates raw child outputs; the child owns the line
-  format. Options: `header`, `separator` (default `'\n'`).
-- **`entries`** — NDJSON dump of `{ index, status, output }`, one entry per
-  line. The format-agnostic view of the batch, rejected children included by
-  default.
-- **`csv`** — projects object outputs into RFC 4180 escaped columns. Options:
-  `columns` (required; each entry is `{ key, header? }` where `key` selects
-  the value on the child output — a dot-path like `'meta.total'` reaches
-  nested fields, matching the `where` semantics; a literal top-level key that
-  contains dots wins over the nested path — and `header` is the label on the
-  header row — a plain string is shorthand for `{ key }`), `delimiter`
-  (default `','`), `header` (default `true`; emits the column labels as the
-  first row).
-
-### Status filter
-
-`status` selects which child results make it into the file, instead of an
-error policy: it defaults to `['fulfilled']` for `lines` and `csv`, and to
-both statuses for `entries`. Filtering is silent by design — a file of
-rejected children for reprocessing is one call away:
-
-```typescript
-const failures = await join(batch, { format: 'entries', status: ['rejected'] })
-```
-
-### Where filter
-
-`where` narrows results with a [sift](https://github.com/crcn/sift.js)
-(MongoDB-style) query over each entry's output, applied after the `status`
-filter. The accepted subset is deliberate — only declarative, JSON-only
-operators: `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`, `$nin`,
-`$exists`, `$not`, `$and`, `$or`, `$nor`, plus plain equality and dot-paths
-for nested fields:
-
-```typescript
-const relevant = await join(batch, {
-  format: 'csv',
-  columns: ['id', 'total'],
-  where: {
-    valid: true, // plain strict equality
-    total: { $gt: 100 }, // operators
-    'meta.situacao': 'aprovada', // dot-path into nested output
-    $or: [{ ref: 'a' }, { ref: 'b' }],
+    return await join(batch, { format: 'lines', header: 'id' })
   },
-})
+)
 ```
 
-`join` also accepts the child output type as a generic — top-level keys then
-narrow to its fields, with values typed by the field (dot-paths are accepted
-without narrowing):
-
-```typescript
-type InvoiceResult = { id: string; valid: boolean; total: number }
-
-await join<InvoiceResult>(batch, {
-  format: 'csv',
-  columns: ['id'],
-  where: { valid: true, total: { $gt: 100 } }, // { valid: 'yes' } rejects
-})
-```
+## Documentation
 
-Anything that cannot serialize losslessly into the workflow history is
-rejected eagerly at `toInput` with a `TypeError`: functions (including
-`$where`), `RegExp` values (including `$regex`), `undefined`, and operators
-outside the subset — a dropped closure would make the replay filter
-differently than the first execution. For arbitrary predicates, write a
-custom activity in project code (the function then lives in versioned code,
-not in history).
+- [Guide](https://tech4humans-brasil.github.io/framework/fs/)
+- [API reference](https://tech4humans-brasil.github.io/framework/api/fs)
+- [Changelog](./CHANGELOG.md)
 
-Two MongoDB semantics worth knowing: array fields match by containment
-(`{ tags: 'a' }` matches `tags: ['a', 'b']`), and equality against an object
-value compares deeply (never by reference). Comparisons never coerce types.
+## License
 
-Filtering runs inside the existing streaming loop, over outputs that are
-already hydrated one at a time — the query compiles once and adds one
-predicate call per entry, no extra serialization and no change to the memory
-profile. It is applied app-side: every child output is still fetched and
-parsed; `where` decides what enters the file, not what leaves the database.
+MIT

package/dist/activities/FileJoinActivity.d.ts

@@ -32,8 +32,7 @@ export declare class FileJoinActivity extends SyncActivity<readonly [batch: Work
     run({ batch, options, filter, }: FileJoinActivityInput): Promise<Binary>;
     toOutput(output: Binary): Binary;
     private toStatus;
-    private writeHeader;
-    private toLine;
+    private toLines;
 }
 export declare function join<TOutput = unknown>(batch: WorkflowBatchCollectionRef, options: FileJoinOptions<TOutput>): Promise<Binary>;
 export {};

package/dist/activities/FileJoinActivity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,YAAY,EAEZ,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EAGxB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAEvC,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAA;AAEhD,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE1E,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;IACxB,MAAM,CAAC,EAAE,mBAAmB,CAAA;CAC7B,CAAA;AAQD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,0BAA0B,EAAE,OAAO,EAAE,eAAe,CAAC,EACtE,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,0BAA0B,EACjC,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAkBX,GAAG,CAAC,EACf,KAAK,EACL,OAAO,EACP,MAAM,GACP,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IA6BnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAIvC,OAAO,CAAC,QAAQ;YAaF,WAAW;IAazB,OAAO,CAAC,MAAM;CAef;AAED,wBAAsB,IAAI,CAAC,OAAO,GAAG,OAAO,EAC1C,KAAK,EAAE,0BAA0B,EACjC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC,CAMjB"}
+{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,KAAK,MAAM,EAGX,YAAY,EAEZ,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EAGxB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAEvC,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAA;AAEhD,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,OAAO,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE1E,KAAK,mBAAmB,CAAC,OAAO,IAAI;IAClC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,KAAK,oBAAoB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAClE,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IACpE,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,KAAK,kBAAkB,CAAC,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG;IAChE,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,GAAG,OAAO,IACzC,oBAAoB,CAAC,OAAO,CAAC,GAC7B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,kBAAkB,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;IACxB,MAAM,CAAC,EAAE,mBAAmB,CAAA;CAC7B,CAAA;AA2CD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,0BAA0B,EAAE,OAAO,EAAE,eAAe,CAAC,EACtE,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,0BAA0B,EACjC,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAeX,GAAG,CAAC,EACf,KAAK,EACL,OAAO,EACP,MAAM,GACP,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAkBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAIvC,OAAO,CAAC,QAAQ;YAWD,OAAO;CAWvB;AAED,wBAAsB,IAAI,CAAC,OAAO,GAAG,OAAO,EAC1C,KAAK,EAAE,0BAA0B,EACjC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC,CAEjB"}

package/dist/activities/FileJoinActivity.js

@@ -1,13 +1,30 @@
-import { once } from 'node:events';
-import { PassThrough } from 'node:stream';
+import { getDefaultHighWaterMark, Readable } from 'node:stream';
 import { Claim, History, SyncActivity, WorkflowBatchClaim, WorkflowBatchQuery, } from '@t4h.framework/core';
 import { FileSystemClaim } from '../FileSystemClaim.js';
 import { CSV } from '../helpers/CSV.js';
-const extensions = {
-    lines: 'txt',
-    entries: 'ndjson',
-    csv: 'csv',
+const formats = {
+    lines: {
+        extension: 'txt',
+        status: ['fulfilled'],
+        header: ({ header, separator = '\n' }) => header && header + separator,
+        line: ({ output }, { separator = '\n' }) => (typeof output === 'string' ? output : JSON.stringify(output)) +
+            separator,
+    },
+    entries: {
+        extension: 'ndjson',
+        status: ['fulfilled', 'rejected'],
+        line: entry => JSON.stringify(entry) + '\n',
+    },
+    csv: {
+        extension: 'csv',
+        status: ['fulfilled'],
+        header: ({ header, columns, delimiter }) => header === false ? undefined : CSV.toHeader(columns, { delimiter }),
+        line: ({ output }, { columns, delimiter }) => CSV.toLine(output, columns, { delimiter }),
+    },
 };
+function formatOf(options) {
+    return formats[options.format];
+}
 export class FileJoinActivity extends SyncActivity {
     claims = new Claim({
         fs: FileSystemClaim,
@@ -16,30 +33,20 @@ export class FileJoinActivity extends SyncActivity {
     toInput(batch, options) {
         if (options.where)
             WorkflowBatchQuery.assertSerializable(options.where, 'where', 'Join where');
-        const filter = {
-            status: batch.status,
-            where: batch.where,
-        };
-        if (!filter.status && !filter.where)
+        const { status, where } = batch;
+        if (!status && !where)
             return { batch: batch.id, options };
-        return { batch: batch.id, options, filter };
+        return { batch: batch.id, options, filter: { status, where } };
     }
     async run({ batch, options, filter, }) {
-        const stream = new PassThrough();
-        const writing = this.claims.fs.write(`join/${this.id}.${extensions[options.format]}`, stream);
-        const write = async (line) => {
-            if (!stream.write(line))
-                await once(stream, 'drain');
-        };
         const entries = WorkflowBatchQuery.filter(this.claims.batch.outputs(batch), WorkflowBatchQuery.merge(filter, {
             status: this.toStatus(options, filter),
             where: options.where,
         }));
-        await this.writeHeader(options, write);
-        for await (const entry of entries)
-            await write(this.toLine(entry, options));
-        stream.end();
-        return await writing;
+        return await this.claims.fs.write(`join/${this.id}.${formatOf(options).extension}`, Readable.from(this.toLines(entries, options), {
+            objectMode: false,
+            highWaterMark: getDefaultHighWaterMark(false),
+        }));
     }
     toOutput(output) {
         return output;
@@ -49,27 +56,15 @@ export class FileJoinActivity extends SyncActivity {
             return options.status;
         if (filter?.status)
             return;
-        if (options.format === 'entries')
-            return ['fulfilled', 'rejected'];
-        return ['fulfilled'];
+        return formatOf(options).status;
     }
-    async writeHeader(options, write) {
-        if (options.format === 'lines' && options.header)
-            await write(options.header + (options.separator ?? '\n'));
-        if (options.format === 'csv' && options.header !== false)
-            await write(CSV.toHeader(options.columns, { delimiter: options.delimiter }));
-    }
-    toLine(entry, options) {
-        if (options.format === 'entries')
-            return JSON.stringify(entry) + '\n';
-        if (options.format === 'csv')
-            return CSV.toLine(entry.output, options.columns, {
-                delimiter: options.delimiter,
-            });
-        const value = typeof entry.output === 'string'
-            ? entry.output
-            : JSON.stringify(entry.output);
-        return value + (options.separator ?? '\n');
+    async *toLines(entries, options) {
+        const format = formatOf(options);
+        const header = format.header?.(options);
+        if (header)
+            yield header;
+        for await (const entry of entries)
+            yield format.line(entry, options);
     }
 }
 export async function join(batch, options) {

package/dist/activities/FileJoinActivity.js.map

@@ -1 +1 @@
-{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAEL,KAAK,EACL,OAAO,EACP,YAAY,EACZ,kBAAkB,EAIlB,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAuCvC,MAAM,UAAU,GAA8C;IAC5D,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,QAAQ;IACjB,GAAG,EAAE,KAAK;CACX,CAAA;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,KAAiC,EACjC,OAAwB;QAExB,IAAI,OAAO,CAAC,KAAK;YACf,kBAAkB,CAAC,kBAAkB,CACnC,OAAO,CAAC,KAAK,EACb,OAAO,EACP,YAAY,CACb,CAAA;QAEH,MAAM,MAAM,GAAwB;YAClC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAA;QAED,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;QAExE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;IAC7C,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EACf,KAAK,EACL,OAAO,EACP,MAAM,GACgB;QACtB,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAA;QAEhC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAClC,QAAQ,IAAI,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC/C,MAAM,CACP,CAAA;QAED,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAChC,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE;YAC/B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACtC,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CACH,CAAA;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAEtC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO;YAAE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;QAE3E,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,QAAQ,CACd,OAAwB,EACxB,MAA4B;QAE5B,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAA;QAEzC,IAAI,MAAM,EAAE,MAAM;YAAE,OAAM;QAE1B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;QAElE,OAAO,CAAC,WAAW,CAAC,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAwB,EACxB,KAAsC;QAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM;YAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAA;QAE3D,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YACtD,MAAM,KAAK,CACT,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAChE,CAAA;IACL,CAAC;IAEO,MAAM,CAAC,KAA0B,EAAE,OAAwB;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;QAErE,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK;YAC1B,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAA;QAEJ,MAAM,KAAK,GACT,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;YAC9B,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAElC,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAA;IAC5C,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,KAAiC,EACjC,OAAiC;IAEjC,OAAO,MAAM,OAAO,CAAC,UAAU,CAC7B,gBAAgB,EAChB,KAAK,EACL,OAA0B,CAC3B,CAAA;AACH,CAAC"}
+{"version":3,"file":"FileJoinActivity.js","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE/D,OAAO,EAEL,KAAK,EACL,OAAO,EACP,YAAY,EACZ,kBAAkB,EAIlB,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AAoDvC,MAAM,OAAO,GAAgB;IAC3B,KAAK,EAAE;QACL,SAAS,EAAE,KAAK;QAChB,MAAM,EAAE,CAAC,WAAW,CAAC;QACrB,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,MAAM,GAAG,SAAS;QACtE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,GAAG,IAAI,EAAE,EAAE,EAAE,CACzC,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9D,SAAS;KACZ;IACD,OAAO,EAAE;QACP,SAAS,EAAE,QAAQ;QACnB,MAAM,EAAE,CAAC,WAAW,EAAE,UAAU,CAAC;QACjC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI;KAC5C;IACD,GAAG,EAAE;QACH,SAAS,EAAE,KAAK;QAChB,MAAM,EAAE,CAAC,WAAW,CAAC;QACrB,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,CACzC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC;QACrE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,CAC3C,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC;KAC7C;CACF,CAAA;AAED,SAAS,QAAQ,CAAC,OAAwB;IACxC,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAgC,CAAA;AAC/D,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAKrC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,kBAAkB;KAC1B,CAAC,CAAA;IAEK,OAAO,CACZ,KAAiC,EACjC,OAAwB;QAExB,IAAI,OAAO,CAAC,KAAK;YACf,kBAAkB,CAAC,kBAAkB,CACnC,OAAO,CAAC,KAAK,EACb,OAAO,EACP,YAAY,CACb,CAAA;QAEH,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAA;QAE/B,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;QAE1D,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAA;IAChE,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EACf,KAAK,EACL,OAAO,EACP,MAAM,GACgB;QACtB,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAChC,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE;YAC/B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACtC,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CACH,CAAA;QAED,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAC/B,QAAQ,IAAI,CAAC,EAAE,IAAI,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAChD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;YAC5C,UAAU,EAAE,KAAK;YACjB,aAAa,EAAE,uBAAuB,CAAC,KAAK,CAAC;SAC9C,CAAC,CACH,CAAA;IACH,CAAC;IAEM,QAAQ,CAAC,MAAc;QAC5B,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,QAAQ,CACd,OAAwB,EACxB,MAA4B;QAE5B,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAA;QAEzC,IAAI,MAAM,EAAE,MAAM;YAAE,OAAM;QAE1B,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,CAAA;IACjC,CAAC;IAEO,KAAK,CAAC,CAAC,OAAO,CACpB,OAA2C,EAC3C,OAAwB;QAExB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAA;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAA;QAEvC,IAAI,MAAM;YAAE,MAAM,MAAM,CAAA;QAExB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO;YAAE,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IACtE,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,KAAiC,EACjC,OAAiC;IAEjC,OAAO,MAAM,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;AACnE,CAAC"}

package/dist/activities/FileReadActivity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAID,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;YAoBN,OAAO;CA8BtB;AAED,wBAAsB,IAAI,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAEjB"}
+{"version":3,"file":"FileReadActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileReadActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAkB,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AA0BD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,EACnD,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GACR,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAcnC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,KAAK;YAYL,MAAM;YAeN,OAAO;CA2BtB;AAED,wBAAsB,IAAI,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAEjB"}

package/dist/activities/FileReadActivity.js

@@ -2,6 +2,18 @@ import { buffer } from 'node:stream/consumers';
 import { Claim, History, SyncActivity } from '@t4h.framework/core';
 import { FileSystemClaim } from '../FileSystemClaim.js';
 const FILE_READ_MAX_BYTES = 1024 * 1024;
+function assertRange(range, contentLength) {
+    const length = range.end - range.start;
+    if (!Number.isInteger(range.start) ||
+        !Number.isInteger(range.end) ||
+        range.start < 0 ||
+        length < 0)
+        throw new TypeError('Read range must be a non-negative byte window');
+    if (range.end > contentLength)
+        throw new TypeError('Read range must not exceed the binary length');
+    if (length > FILE_READ_MAX_BYTES)
+        throw new RangeError(`Length ${String(length)} exceeded max ${String(FILE_READ_MAX_BYTES)}`);
+}
 export class FileReadActivity extends SyncActivity {
     claims = new Claim({
         fs: FileSystemClaim,
@@ -12,13 +24,7 @@ export class FileReadActivity extends SyncActivity {
     async run({ binary, options, }) {
         const encoding = options.encoding ?? 'utf8';
         const range = options.range ?? { start: 0, end: binary.contentLength };
-        const length = range.end - range.start;
-        if (!Number.isInteger(range.start) || range.start < 0 || length < 0)
-            throw new TypeError('Read range must be a non-negative byte window');
-        if (range.end > binary.contentLength)
-            throw new TypeError('Read range must not exceed the binary length');
-        if (length > FILE_READ_MAX_BYTES)
-            throw new RangeError(`Length ${String(length)} exceeded max ${String(FILE_READ_MAX_BYTES)}`);
+        assertRange(range, binary.contentLength);
         const slice = await this.slice(binary, range, encoding);
         if (slice)
             return (await buffer(await slice.stream())).toString(encoding);
@@ -36,21 +42,19 @@ export class FileReadActivity extends SyncActivity {
         }
     }
     async window(readable, range) {
-        let collected;
         try {
-            collected = await this.collect(readable, range);
+            const content = Buffer.concat(await this.collect(readable, range));
+            if (content.length < range.end - range.start)
+                throw new RangeError('Binary stream ended before the read window was filled');
+            return content;
         }
         finally {
             readable.destroy();
         }
-        if (collected.size < range.end - range.start)
-            throw new RangeError('Binary stream ended before the read window was filled');
-        return Buffer.concat(collected.chunks);
     }
     async collect(readable, range) {
         const chunks = [];
         let position = 0;
-        let size = 0;
         for await (const data of readable) {
             const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data);
             const chunkStart = position;
@@ -60,11 +64,10 @@ export class FileReadActivity extends SyncActivity {
             const from = Math.max(range.start - chunkStart, 0);
             const to = Math.min(range.end - chunkStart, chunk.length);
             chunks.push(chunk.subarray(from, to));
-            size += to - from;
             if (position >= range.end)
                 break;
         }
-        return { chunks, size };
+        return chunks;
     }
 }
 export async function read(binary, options = {}) {