@t4h.framework/fs 0.5.0 → 0.6.0

package/.ai/skills/framework-fs/SKILL.md

@@ -19,10 +19,16 @@ Package path: `framework/packages/fs`. Peer dependency: `@t4h.framework/core` (`
 
 ## Import surface
 
-The package exports a single public symbol from `src/fs.ts`:
+Public exports live in `src/fs.ts`:
 
 ```typescript
-import { FileSystemClaim } from '@t4h.framework/fs'
+import {
+  FileSystemClaim,
+  // file fan-out / fold — see the framework-fs-split skill
+  split, Split, FileSplitActivity, type FileSplitOptions,
+  join, FileJoinActivity, type FileJoinOptions,
+  ByteDelimiterScanner, type FileSplitDelimiter, type ByteRange,
+} from '@t4h.framework/fs'
 ```
 
 Do not import from internal paths such as `src/FileSystemClaim.ts` in application code.
@@ -42,6 +48,12 @@ export abstract class FileSystemClaim {
   ): Binary | Promise<Binary>
 
   public abstract read(path: string): Binary | Promise<Binary>
+
+  public abstract slice(
+    binary: Binary,
+    range: { start: number; end: number },
+    encoding?: BufferEncoding,
+  ): Binary | Promise<Binary>
 }
 ```
 
@@ -49,8 +61,9 @@ export abstract class FileSystemClaim {
 |--------|---------|
 | `write(path, readable)` | Persist stream or buffer at `path`; return a `Binary` handle (metadata + `stream()`). |
 | `read(path)` | Load an existing file at `path`; return a `Binary` handle. |
+| `slice(binary, range, encoding?)` | A view over a byte range of an existing binary, **without copying data**. When `encoding` is set, the runtime hydrates the slice as a decoded string instead of a binary — this is how `split` children receive plain strings. |
 
-Both methods may return synchronously or as a `Promise`. Implementations choose storage layout; callers should only rely on the returned `Binary` API.
+All methods may return synchronously or as a `Promise`. Implementations choose storage layout; callers should only rely on the returned `Binary` API.
 
 ### Binary (from `@t4h.framework/core`)
 
@@ -104,6 +117,7 @@ These packages depend on `@t4h.framework/fs` and declare `FileSystemClaim`; read
 |---------|--------|
 | `@t4h.framework/http` | The HTTP activity persists response bodies to `fs` and returns them wrapped in a `Body` — you read them with `res.body.json()` / `text()`. |
 | `@t4h.framework/jsx` | Render activities write their output to `fs` and hand back a `Binary` you consume in the workflow. |
+| `@t4h.framework/fs` itself | `FileSplitActivity` persists the durable copy and slices it by ranges; `FileJoinActivity` streams batch results into a new file. See the `framework-fs-split` skill. |
 
 Orchestrate these package primitives from your workflow; the `fs` write/read happens inside them.
 
@@ -186,6 +200,7 @@ Assert `mockWrite` was called with the expected path and body when relevant.
 
 ## Related skills
 
+- `framework-fs-split` — `split`/`join`: fan a file out into a batch of workflows and fold the results back into a file.
 - `framework-core` — `Claim`, `Binary`, and the `TestWorkflowEnvironment` testing harness.
 - `framework-http` — HTTP activities that persist bodies via `FileSystemClaim`.
 - `framework-jsx` — render activities that write/read via `fs`.

package/.ai/skills/framework-fs-split/SKILL.md

@@ -0,0 +1,255 @@
+---
+name: framework-fs-split
+description: >-
+  Guides correct use of `split` and `join` from @t4h.framework/fs: fanning one
+  large file out into a batch of workflows (one per byte-range slice) and
+  folding the batch results back into a single file — with O(1) parent history
+  and bounded memory regardless of file size. Use whenever a workflow must
+  process a large file record by record — positional files, plain-text lines,
+  NDJSON, batch imports, per-record fan-out with an aggregated output (e.g. a
+  CSV report) — or when the user mentions splitting/chunking a file, running a
+  workflow per line/record, waiting for a batch, collecting batch results, or
+  the symbols `split`, `join`, `Split`, `FileSplitActivity`,
+  `FileJoinActivity`, `WorkflowBatchRef`, `WorkflowBatchClaim`. Prefer this
+  skill over hand-rolling stream loops + `startWorkflow` calls.
+---
+
+# `split` / `join` — file fan-out and fold for workflows
+
+`split` turns **one file into a batch of workflows**: it streams the file once,
+cuts it into **byte-range slices** (never copying slice content), starts one
+child workflow per slice, and **suspends the parent until every child is
+terminal**. `join` is the inverse: it folds the batch results back into a
+single file, streaming one result at a time.
+
+```typescript
+import { split, join, Split } from '@t4h.framework/fs'
+
+const processRecord = new Workflow(
+  { id: 'process-record', schema: Type.String() },
+  async (record: string) => {
+    // record is the raw slice content — no Binary handling needed
+    return toCsvRow(parsePositional(record))
+  },
+)
+
+const main = new Workflow(fileSchema, async (file: Binary) => {
+  const batch = await split(file, processRecord, {
+    delimiter: Split.char('\n10', { next: 2 }),
+    timeout: 3_600_000,
+  })
+  // this line only runs once the WHOLE batch is terminal;
+  // batch: WorkflowBatchRef — { id: string, count: number }
+
+  return await join(batch, { format: 'lines', header: 'id;value;date' })
+  // Binary: the aggregated file — no slice ever crossed workflow memory
+})
+```
+
+Package path: `framework/packages/fs`. Import only from the package root.
+
+---
+
+## The memory model (the golden rule)
+
+Everything here exists to keep the parent history **O(1)** whatever the file
+size:
+
+1. `split` reads the source stream **exactly once, worker side**: the same
+   pass persists a durable copy through `FileSystemClaim.write` while a byte
+   scanner records slice offsets. No slice content is materialized.
+2. Children receive **range views** over the durable copy
+   (`FileSystemClaim.slice`); the runtime hydrates each slice only inside that
+   child's own run — as a plain decoded string when the child's schema is
+   `Type.String()`.
+3. The parent resumes with only `WorkflowBatchRef { id, count }`. Per-child
+   results are **never aggregated onto the ref**: they live with each child
+   run.
+4. `join` streams results one at a time through `WorkflowBatchClaim.outputs`
+   into `FileSystemClaim.write` with backpressure; only the resulting `Binary`
+   reaches history.
+
+**Golden rule:** volume proportional to the batch size only ever flows inside
+an activity, through a claim. Into workflow history goes a fold result or a
+bounded slice — never the N items.
+
+---
+
+## `split`
+
+```typescript
+async function split(
+  binary: Binary,
+  workflow: Workflow<any, any>,
+  options: FileSplitOptions,
+): Promise<WorkflowBatchRef>
+```
+
+The child workflow declares its **real input type** (e.g. `Type.String()`) —
+it never sees `Binary`, ranges, or storage. `await split(...)` suspends the
+parent (one pending activity, one resume, one replay) until every child is
+terminal — **regardless of each child succeeding or failing**: a rejected
+child never rejects the batch (allSettled semantics). An empty batch resolves
+immediately.
+
+### Delimiters (`options.delimiter`)
+
+The cut is a serializable spec. When the delimiter is found at
+`[p, p + length)`, the previous slice ends at `p + keep.previous` and the next
+slice starts at `p + length - keep.next`; the bytes in between are discarded
+(defaults discard the whole delimiter — classic line semantics).
+
+```typescript
+Split.line()                     // cut at every '\n', delimiter discarded
+Split.char(';')                  // cut at any byte sequence
+Split.char(';', { previous: 1 }) // keep the ';' with the previous slice
+Split.char('\n10', { next: 2 })  // positional files, see below
+```
+
+Positional files where a logical record spans multiple physical lines and a
+new record only starts when a line begins with a marker (the record type
+code):
+
+```
+10JOHN;123\n20detail-a\n20detail-b\n10MARY;456\n20detail-c
+└──────────── record 1 ───────────┘└─────── record 2 ──────┘
+```
+
+`Split.char('\n10', { next: 2 })`: the newline between records is discarded
+and the `10` marker — which is **data**, not separator — stays at the start of
+the next slice. Forgetting `{ next: 2 }` silently strips the record type code
+from every slice.
+
+Offsets are always computed over **raw bytes** (never decoded characters), so
+multi-byte encodings cannot skew the ranges a ranged download will use.
+
+### Other options
+
+| Option | Default | Behavior |
+|--------|---------|----------|
+| `every` | `1` | Cut only every N delimiter occurrences — groups N records per child. One million lines with `every: 10_000` start 100 children, each receiving a raw block (inner delimiters untouched) to iterate locally. `keep` arithmetic applies only at actual cut points. |
+| `skipEmpty` | `true` | Drops zero-length slices (consecutive delimiters, file starting/ending with one) so they never become child workflows. Turn off when slice index must align 1:1 with record position. "Empty" means zero bytes — a whitespace-only line is a regular slice. |
+| `encoding` | `'utf8'` | Converts the delimiter to bytes and tells the runtime how to hydrate each slice for the child (string when set on the slice). |
+| `timeout` | none | Max milliseconds to await the batch; on expiry the activity **rejects** (reason `timeout`) instead of suspending forever — the safety net for a child that never reaches a terminal state. |
+
+---
+
+## `join`
+
+```typescript
+async function join(
+  batch: WorkflowBatchRef,
+  options: FileJoinOptions,
+): Promise<Binary>
+```
+
+Formats (content is the **child's** job — the child returns the final line;
+`join` only owns structure):
+
+| Format | Behavior |
+|--------|----------|
+| `lines` | Concatenates raw child outputs, one per line. Options: `header`, `separator` (default `'\n'`). Non-string outputs are JSON-stringified. |
+| `entries` | NDJSON dump of `{ index, status, output }`, one per line — the format-agnostic view of the batch, rejected children included by default. |
+| `csv` | Projects object outputs into RFC 4180 escaped columns. Options: `columns` (required; entries are `{ key, header? }` — `key` selects the value on the child output, `header` labels the header row, plain string is shorthand for `{ key }`), `delimiter` (default `','`), `header` (default `true`). Missing keys become empty cells; nested values are JSON inside the cell. Value *formatting* belongs in the child workflow (sandboxed user code); the join only owns layout. |
+
+**Status filter** (`status?: ('fulfilled' | 'rejected')[]`) selects which
+results make it into the file — there is no error policy. Defaults:
+`['fulfilled']` for `lines`/`csv`, both for `entries`. Filtering is silent by
+design; a reprocessing file of failures is one call away:
+
+```typescript
+const failures = await join(batch, { format: 'entries', status: ['rejected'] })
+```
+
+---
+
+## Runtime requirements
+
+- Claim providers: `FileSystemClaim` (with `slice`), `WorkflowClaim` (with
+  `startMany`), and `WorkflowBatchClaim` (from `@t4h.framework/core`).
+- Production whitelist: `FileSplitActivity`, `FileJoinActivity` and core's
+  `StartManyWorkflowsActivity` must be registered via `runtime.addActivity`.
+- `FileSplitActivity` is an **AsyncActivity**: the heavy work happens in
+  `bootstrap`, executed worker side atomically with the pending suspension —
+  children cannot finish before the parent is awaiting them.
+- `FileJoinActivity` is a **SyncActivity**: batches are only consumed after
+  the completion barrier resolved, so there is nothing to await.
+
+---
+
+## Testing
+
+Drive `bootstrap`/`run` directly with fake claims inside `Claim.run` (patterns
+from `src/activities/__tests__/`):
+
+```typescript
+import { AsyncActivityResultPending, Claim, WorkflowBatchClaim, WorkflowBatchRef, WorkflowClaim } from '@t4h.framework/core'
+import { FileSplitActivity, FileSystemClaim, Split } from '@t4h.framework/fs'
+
+const write = vi.fn(async (_path, readable) => durableBinary) // must consume the stream
+const slice = vi.fn((binary, range, encoding) => ({ binary, range, encoding }))
+const startMany = vi.fn(async (_wf, inputs) => new WorkflowBatchRef('batch-1', inputs.length))
+
+const result = await Claim.run(
+  [
+    { provide: FileSystemClaim, value: { write, slice } as never },
+    { provide: WorkflowClaim, value: { startMany } as never },
+  ],
+  () => {
+    const activity = new FileSplitActivity()
+    activity.id = 'activity-1'
+    return activity.bootstrap({ binary, workflow: 'child', options: { delimiter: Split.line() } })
+  },
+)
+
+expect(result).toBeInstanceOf(AsyncActivityResultPending)
+expect(slice).toHaveBeenCalledWith(durableBinary, { start: 0, end: 3 }, 'utf8')
+```
+
+For `join`, provide `WorkflowBatchClaim` as an async generator:
+
+```typescript
+const outputs = vi.fn(async function* () {
+  yield { index: 0, status: 'fulfilled', output: 'line-1' }
+})
+```
+
+Behaviors worth asserting because they regress easily: the durable copy keeps
+the original bytes; slices are views over the durable copy (not the source);
+`keep` arithmetic on positional cuts; the empty batch resolving immediately;
+CSV escaping (quotes doubled, delimiter/newline wrapped); the status filter.
+
+---
+
+## Anti-patterns
+
+- **Paging batch results into the workflow.** A `readBatch(ref, cursor)` loop
+  in workflow code turns every page into a replayed history entry — O(N) is
+  back. Aggregation belongs inside `join` (or a framework activity consuming
+  `WorkflowBatchClaim`).
+- **Expecting per-child outputs on the ref.** `WorkflowBatchRef` is
+  deliberately `{ id, count }` only. Results are read through `join` — e.g.
+  `format: 'entries'` for the faithful dump.
+- **Treating a rejected child as a batch failure.** The batch always resolves
+  (allSettled). Decide what failures mean: compare against `batch.count`,
+  generate the `status: ['rejected']` file, or tolerate.
+- **Baking record format into `join`.** The child returns the final line
+  (`lines`) or a projectable object (`csv`); custom formats live in the child
+  workflow — sandboxed user code — not in new join options.
+- **Splitting when you need the file whole.** If downstream wants the blob
+  intact (hash it, upload it), pass the `Binary` through.
+- **Forgetting `keep` on positional delimiters.** `Split.char('\n10')` without
+  `{ next: 2 }` silently discards the record type code from every slice.
+- **Hand-rolling the loop.** A stream-and-`startWorkflow` loop inside an
+  activity re-parses on replay, misses the barrier, and pushes N entries into
+  history.
+
+---
+
+## Related skills
+
+- `framework-fs` — `FileSystemClaim` contract (including `slice`), `Binary`,
+  provider wiring.
+- `framework-core` — `startWorkflowMany` (the array-input sibling of `split`),
+  `WorkflowBatchRef`, `WorkflowBatchClaim`, `Claim.run`, activity model,
+  testing harness.

package/README.md

@@ -1,6 +1,7 @@
 # @t4h.framework/fs
 
-File system claim for reading and writing files within activities.
+File system claim for reading and writing files within activities, and a file
+split activity that fans a large file out into one child workflow per slice.
 
 ## Claim
 
@@ -16,8 +17,17 @@ import { FileSystemClaim } from '@t4h.framework/fs'
 
 ```typescript
 abstract class FileSystemClaim {
-  public abstract write(path: string, readable: Readable | Buffer): Binary | Promise<Binary>
+  public abstract write(
+    path: string,
+    readable: Readable | Buffer,
+  ): Binary | Promise<Binary>
   public abstract read(path: string): Binary | Promise<Binary>
+
+  public abstract slice(
+    binary: Binary,
+    range: { start: number; end: number },
+    encoding?: BufferEncoding,
+  ): Binary | Promise<Binary>
 }
 ```
 
@@ -27,7 +37,11 @@ abstract class FileSystemClaim {
 import { SyncActivity, Claim } from '@t4h.framework/core'
 import { FileSystemClaim } from '@t4h.framework/fs'
 
-class SaveReportActivity extends SyncActivity<[data: Buffer], { path: string }, any> {
+class SaveReportActivity extends SyncActivity<
+  [data: Buffer],
+  { path: string },
+  any
+> {
   private readonly claims = new Claim({
     fs: FileSystemClaim,
   })
@@ -54,3 +68,170 @@ The runtime must supply a concrete implementation:
 ```typescript
 { provide: FileSystemClaim, value: new LocalFileSystemClaimImpl() }
 ```
+
+## File split
+
+### split
+
+Splits a file into byte-range slices, starts one child workflow per slice as
+a single batch, and suspends the parent until every child is terminal —
+regardless of each child succeeding or failing (a rejected child never
+rejects the batch):
+
+```typescript
+import { split, Split } from '@t4h.framework/fs'
+
+const processRecord = new Workflow(
+  { id: 'process-record', schema: Type.String() },
+  async (record: string) => {
+    // record is the raw slice content
+  },
+)
+
+const main = new Workflow(fileSchema, async (file: Binary) => {
+  const batch = await split(file, processRecord, {
+    delimiter: Split.line(),
+  })
+
+  // this line only runs once the whole batch is terminal;
+  // batch: WorkflowBatchRef — { id: string, count: number }
+})
+```
+
+The design goal is constant memory regardless of file size:
+
+1. The source stream is read **exactly once**, worker side. The same pass
+   persists a durable copy through `FileSystemClaim.write` while a byte
+   scanner records the slice offsets — no slice content is ever materialized
+   in the activity.
+2. Children receive **range views** over the durable copy
+   (`FileSystemClaim.slice`).
+3. The activity output is just the batch reference (`{ id, count }`), so the
+   parent history stays O(1) whatever the slice count. Per-child results are
+   never aggregated: they live with each child run, to be inspected
+   individually through the batch id.
+
+Child workflows declare their real input type (e.g. `Type.String()`); how the
+bytes reach them is a runtime concern.
+
+### Delimiters
+
+The cut is defined by the dev through a serializable delimiter spec. When the
+delimiter is found at `[p, p + length)`, the previous slice ends at
+`p + keep.previous` and the next slice starts at `p + length - keep.next`; the
+bytes in between are discarded. The defaults discard the whole delimiter
+(classic line semantics).
+
+```typescript
+Split.line() // cut at every '\n', delimiter discarded
+Split.char(';') // cut at any byte sequence
+Split.char(';', { previous: 1 }) // keep the ';' with the previous slice
+Split.char('\n10', { next: 2 }) // positional files, see below
+```
+
+The last form targets positional files where a logical record spans multiple
+physical lines and a new record only starts when a line begins with a marker
+(the record type code):
+
+```
+10JOHN;123\n20detail-a\n20detail-b\n10MARY;456\n20detail-c
+└──────────── record 1 ───────────┘└─────── record 2 ──────┘
+```
+
+With `Split.char('\n10', { next: 2 })` the newline between records is
+discarded, and the `10` marker — which is data, not separator — stays at the
+start of the next slice.
+
+### Options
+
+```typescript
+await split(file, workflow, {
+  delimiter: Split.line(), // required: where to cut
+  every: 10_000, // cut only every N occurrences (groups records)
+  skipEmpty: false, // keep zero-length slices (default: skipped)
+  encoding: 'latin1', // delimiter bytes + slice hydration (default: utf8)
+  timeout: 3_600_000, // max ms to await the batch; rejects on expiry
+})
+```
+
+- **`every`** groups N records per child. With one million lines, `every:
+10_000` starts 100 children instead of one million; each child receives a
+  raw block with its inner delimiters untouched and iterates locally. `keep`
+  arithmetic only applies at actual cut points.
+- **`skipEmpty`** (default `true`) drops zero-length slices — consecutive
+  delimiters, or a file starting/ending with one — so they never become child
+  workflows. Turn it off when the slice index must align 1:1 with the record
+  position in the source file. "Empty" means zero bytes: a whitespace-only
+  line is a regular slice.
+- **`encoding`** converts the delimiter to bytes and tells the runtime how to
+  hydrate each slice for the child. Offsets are always computed over raw
+  bytes, so multi-byte encodings never skew the ranges.
+- **`timeout`** is the maximum time in milliseconds to await the batch. On
+  expiry the activity rejects (reason `timeout`) instead of suspending
+  forever — the safety net for a child that never reaches a terminal state.
+
+## File join
+
+The inverse of `split`: folds the results of a completed batch into a single
+file. Outputs stream in one at a time through `WorkflowBatchClaim` and flow
+straight into `FileSystemClaim.write` with backpressure, so memory stays
+bounded whatever the batch size — only the resulting binary reaches the
+workflow history.
+
+```typescript
+import { join, split, Split } from '@t4h.framework/fs'
+
+const batch = await split(file, processRecord, { delimiter: Split.line() })
+
+const csv = await join(batch, {
+  format: 'csv',
+  columns: [
+    { key: 'id', header: 'Código' },
+    { key: 'value', header: 'Valor (R$)' },
+    'date', // shorthand: key === header
+  ],
+})
+// csv: Binary
+```
+
+### Formats
+
+- **`lines`** — concatenates raw child outputs; the child owns the line
+  format. Options: `header`, `separator` (default `'\n'`).
+- **`entries`** — NDJSON dump of `{ index, status, output }`, one entry per
+  line. The format-agnostic view of the batch, rejected children included by
+  default.
+- **`csv`** — projects object outputs into RFC 4180 escaped columns. Options:
+  `columns` (required; each entry is `{ key, header? }` where `key` selects
+  the value on the child output and `header` is the label on the header row —
+  a plain string is shorthand for `{ key }`), `delimiter` (default `','`),
+  `header` (default `true`; emits the column labels as the first row).
+
+### Status filter
+
+`status` selects which child results make it into the file, instead of an
+error policy: it defaults to `['fulfilled']` for `lines` and `csv`, and to
+both statuses for `entries`. Filtering is silent by design — a file of
+rejected children for reprocessing is one call away:
+
+```typescript
+const failures = await join(batch, { format: 'entries', status: ['rejected'] })
+```
+
+### ByteDelimiterScanner
+
+The scanner behind `split` is exported for direct use. It is a pass-through
+`Transform`: chunks flow through unchanged (so the same pass can persist the
+file) while it records slice offsets in absolute byte positions, handling
+delimiters split across chunk boundaries. `ranges` is complete once the stream
+has flushed.
+
+```typescript
+import { ByteDelimiterScanner } from '@t4h.framework/fs'
+
+const scanner = new ByteDelimiterScanner({ delimiter: { value: '\n' } })
+
+await pipeline(source, scanner, destination)
+
+scanner.ranges // [{ start: 0, end: 3 }, ...]
+```

package/dist/FileSystemClaim.d.ts

@@ -3,5 +3,9 @@ import type { Binary } from '@t4h.framework/core';
 export declare abstract class FileSystemClaim {
     abstract write(path: string, readable: Readable | Buffer): Binary | Promise<Binary>;
     abstract read(path: string): Binary | Promise<Binary>;
+    abstract slice(binary: Binary, range: {
+        start: number;
+        end: number;
+    }, encoding?: BufferEncoding): Binary | Promise<Binary>;
 }
 //# sourceMappingURL=FileSystemClaim.d.ts.map

package/dist/FileSystemClaim.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FileSystemClaim.d.ts","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAEjD,8BAAsB,eAAe;aACnB,KAAK,CACnB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAEX,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAC7D"}
+{"version":3,"file":"FileSystemClaim.d.ts","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAEjD,8BAAsB,eAAe;aACnB,KAAK,CACnB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAEX,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;aAE5C,KAAK,CACnB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EACrC,QAAQ,CAAC,EAAE,cAAc,GACxB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAC5B"}

package/dist/FileSystemClaim.js.map

@@ -1 +1 @@
-{"version":3,"file":"FileSystemClaim.js","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,eAAe;CAOpC"}
+{"version":3,"file":"FileSystemClaim.js","sourceRoot":"","sources":["../src/FileSystemClaim.ts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,eAAe;CAapC"}

package/dist/activities/FileJoinActivity.d.ts

@@ -0,0 +1,39 @@
+import { type Binary, SyncActivity, type WorkflowBatchRef } from '@t4h.framework/core';
+export type FileJoinStatus = 'fulfilled' | 'rejected';
+type FileJoinOptionsBase = {
+    status?: FileJoinStatus[];
+};
+type FileJoinOptionsLines = FileJoinOptionsBase & {
+    format: 'lines';
+    header?: string;
+    separator?: string;
+};
+type FileJoinOptionsEntries = FileJoinOptionsBase & {
+    format: 'entries';
+};
+export type FileJoinColumn = string | {
+    key: string;
+    header?: string;
+};
+type FileJoinOptionsCsv = FileJoinOptionsBase & {
+    format: 'csv';
+    columns: FileJoinColumn[];
+    delimiter?: string;
+    header?: boolean;
+};
+export type FileJoinOptions = FileJoinOptionsLines | FileJoinOptionsEntries | FileJoinOptionsCsv;
+export type FileJoinActivityInput = {
+    batch: string;
+    options: FileJoinOptions;
+};
+export declare class FileJoinActivity extends SyncActivity<readonly [batch: WorkflowBatchRef, options: FileJoinOptions], FileJoinActivityInput, Binary, Binary> {
+    private readonly claims;
+    toInput(batch: WorkflowBatchRef, options: FileJoinOptions): FileJoinActivityInput;
+    run({ batch, options }: FileJoinActivityInput): Promise<Binary>;
+    toOutput(output: Binary): Binary;
+    private writeHeader;
+    private toLine;
+}
+export declare function join(batch: WorkflowBatchRef, options: FileJoinOptions): Promise<Binary>;
+export {};
+//# sourceMappingURL=FileJoinActivity.d.ts.map

package/dist/activities/FileJoinActivity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileJoinActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileJoinActivity.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EAGX,YAAY,EAGZ,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAA;AAI5B,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,CAAA;AAErD,KAAK,mBAAmB,GAAG;IACzB,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;CAC1B,CAAA;AAED,KAAK,oBAAoB,GAAG,mBAAmB,GAAG;IAChD,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,sBAAsB,GAAG,mBAAmB,GAAG;IAClD,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,cAAc,GACtB,MAAM,GACN;IACE,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAEL,KAAK,kBAAkB,GAAG,mBAAmB,GAAG;IAC9C,MAAM,EAAE,KAAK,CAAA;IACb,OAAO,EAAE,cAAc,EAAE,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,GACvB,oBAAoB,GACpB,sBAAsB,GACtB,kBAAkB,CAAA;AAEtB,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,eAAe,CAAA;CACzB,CAAA;AAgCD,qBAAa,gBAAiB,SAAQ,YAAY,CAChD,SAAS,CAAC,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,eAAe,CAAC,EAC5D,qBAAqB,EACrB,MAAM,EACN,MAAM,CACP;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAGrB;IAEK,OAAO,CACZ,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,GACvB,qBAAqB;IAIX,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;IAgCrE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;YAIzB,WAAW;IAkBzB,OAAO,CAAC,MAAM;CAsBf;AAED,wBAAsB,IAAI,CACxB,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,MAAM,CAAC,CAEjB"}

package/dist/activities/FileJoinActivity.js

@@ -0,0 +1,87 @@
+import { once } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { Claim, History, SyncActivity, WorkflowBatchClaim, } from '@t4h.framework/core';
+import { FileSystemClaim } from '../FileSystemClaim.js';
+const extensions = {
+    lines: 'txt',
+    entries: 'ndjson',
+    csv: 'csv',
+};
+function columnKey(column) {
+    return typeof column === 'string' ? column : column.key;
+}
+function columnHeader(column) {
+    return typeof column === 'string' ? column : (column.header ?? column.key);
+}
+function csvEscape(value, delimiter) {
+    if (value === null || typeof value === 'undefined')
+        return '';
+    const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
+    if (text.includes(delimiter) ||
+        text.includes('"') ||
+        text.includes('\n') ||
+        text.includes('\r'))
+        return `"${text.replaceAll('"', '""')}"`;
+    return text;
+}
+export class FileJoinActivity extends SyncActivity {
+    claims = new Claim({
+        fs: FileSystemClaim,
+        batch: WorkflowBatchClaim,
+    });
+    toInput(batch, options) {
+        return { batch: batch.id, options };
+    }
+    async run({ batch, options }) {
+        const stream = new PassThrough();
+        const writing = this.claims.fs.write(`join/${this.id}.${extensions[options.format]}`, stream);
+        const write = async (line) => {
+            if (!stream.write(line))
+                await once(stream, 'drain');
+        };
+        const statuses = new Set(options.status ??
+            (options.format === 'entries'
+                ? ['fulfilled', 'rejected']
+                : ['fulfilled']));
+        await this.writeHeader(options, write);
+        for await (const entry of this.claims.batch.outputs(batch)) {
+            if (!statuses.has(entry.status))
+                continue;
+            await write(this.toLine(entry, options));
+        }
+        stream.end();
+        return await writing;
+    }
+    toOutput(output) {
+        return output;
+    }
+    async writeHeader(options, write) {
+        if (options.format === 'lines' && options.header)
+            await write(options.header + (options.separator ?? '\n'));
+        if (options.format === 'csv' && options.header !== false) {
+            const delimiter = options.delimiter ?? ',';
+            await write(options.columns
+                .map(column => csvEscape(columnHeader(column), delimiter))
+                .join(delimiter) + '\n');
+        }
+    }
+    toLine(entry, options) {
+        if (options.format === 'entries')
+            return JSON.stringify(entry) + '\n';
+        if (options.format === 'csv') {
+            const delimiter = options.delimiter ?? ',';
+            const output = (entry.output ?? {});
+            return (options.columns
+                .map(column => csvEscape(output[columnKey(column)], delimiter))
+                .join(delimiter) + '\n');
+        }
+        const value = typeof entry.output === 'string'
+            ? entry.output
+            : JSON.stringify(entry.output);
+        return value + (options.separator ?? '\n');
+    }
+}
+export async function join(batch, options) {
+    return await History.reconciler(FileJoinActivity, batch, options);
+}
+//# sourceMappingURL=FileJoinActivity.js.map