@t4h.framework/fs 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +54 -112
- package/dist/activities/FileSplitActivity.d.ts +36 -0
- package/dist/activities/FileSplitActivity.d.ts.map +1 -0
- package/dist/activities/FileSplitActivity.js +46 -0
- package/dist/activities/FileSplitActivity.js.map +1 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.d.ts +2 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.d.ts.map +1 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.js +167 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.js.map +1 -0
- package/dist/helpers/__tests__/csv-reader.spec.d.ts +2 -0
- package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +1 -0
- package/dist/helpers/__tests__/csv-reader.spec.js +60 -0
- package/dist/helpers/__tests__/csv-reader.spec.js.map +1 -0
- package/dist/helpers/csv-reader.d.ts +8 -0
- package/dist/helpers/csv-reader.d.ts.map +1 -0
- package/dist/helpers/csv-reader.js +14 -0
- package/dist/helpers/csv-reader.js.map +1 -0
- package/dist/helpers/parsers.d.ts +9 -0
- package/dist/helpers/parsers.d.ts.map +1 -0
- package/dist/helpers/parsers.js +14 -0
- package/dist/helpers/parsers.js.map +1 -0
- package/dist/testing/index.d.ts +3 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +32 -0
- package/dist/testing/index.js.map +1 -0
- package/package.json +3 -3
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
name: framework-fs
|
|
3
3
|
description: >-
|
|
4
4
|
Guides correct use of @t4h.framework/fs in T4H Framework workflows: the
|
|
5
|
-
FileSystemClaim abstraction
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
FileSystemClaim abstraction that the package activities (http, jsx) use to
|
|
6
|
+
read/write files, the Binary result workflows consume, and runtime providers.
|
|
7
|
+
Use when a workflow orchestrates file-backed package activities, consuming
|
|
8
|
+
Binary results, mocking FileSystemClaim in workflow tests, registering fs
|
|
9
|
+
claim providers, or debugging missing fs provider errors.
|
|
10
10
|
---
|
|
11
11
|
|
|
12
12
|
# @t4h.framework/fs
|
|
@@ -62,45 +62,25 @@ export abstract class Binary {
|
|
|
62
62
|
}
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
The package activities return this `Binary` to your workflow — e.g. an `http`
|
|
66
|
+
response `Body` is backed by it. Consume it with `stream()` (or the higher-level
|
|
67
|
+
`body.json()` / `text()` that packages expose):
|
|
66
68
|
|
|
67
69
|
```typescript
|
|
68
|
-
const binary = await this.claims.fs.write('reports/output.pdf', buffer)
|
|
69
70
|
const stream = await binary.stream()
|
|
70
71
|
// consume stream (e.g. stream.toArray() then Buffer.concat)
|
|
71
72
|
```
|
|
72
73
|
|
|
73
74
|
---
|
|
74
75
|
|
|
75
|
-
##
|
|
76
|
+
## How workflows use it
|
|
76
77
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
class SaveReportActivity extends SyncActivity<[data: Buffer], { path: string }, any, any> {
|
|
84
|
-
private readonly claims = new Claim({
|
|
85
|
-
fs: FileSystemClaim,
|
|
86
|
-
})
|
|
87
|
-
|
|
88
|
-
public async toInput(data: Buffer) {
|
|
89
|
-
return { path: 'reports/output.pdf' }
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
public async run(input: { path: string }) {
|
|
93
|
-
const binary = await this.claims.fs.write(input.path, Buffer.from('...'))
|
|
94
|
-
return { contentType: binary.contentType }
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
public toOutput(output: any) {
|
|
98
|
-
return output
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
```
|
|
102
|
-
|
|
103
|
-
Claim key name (`fs` above) is arbitrary but must match how you access `this.claims.fs`. Downstream packages use `fs` consistently (`AbstractHttpRequestActivity`, jsx render activities).
|
|
78
|
+
Workflows never call `FileSystemClaim` directly. File I/O happens inside the
|
|
79
|
+
**ready-made package activities** that already own the claim — the `http` client
|
|
80
|
+
persists response bodies, jsx render activities write their output — and you
|
|
81
|
+
read the resulting `Binary` (e.g. `await res.body.json()` / `text()`). Your only
|
|
82
|
+
responsibility is that the runtime **registers a provider** (below). See
|
|
83
|
+
**Known consumers** for which package primitives to orchestrate.
|
|
104
84
|
|
|
105
85
|
---
|
|
106
86
|
|
|
@@ -122,94 +102,58 @@ These packages depend on `@t4h.framework/fs` and declare `FileSystemClaim`; read
|
|
|
122
102
|
|
|
123
103
|
| Package | Usage |
|
|
124
104
|
|---------|--------|
|
|
125
|
-
| `@t4h.framework/http` |
|
|
126
|
-
| `@t4h.framework/jsx` |
|
|
127
|
-
|
|
128
|
-
### HTTP response persistence (reference)
|
|
129
|
-
|
|
130
|
-
From `AbstractHttpRequestActivity.run`:
|
|
131
|
-
|
|
132
|
-
```typescript
|
|
133
|
-
const ext =
|
|
134
|
-
typeof contentType === 'string' ? mime.getExtension(contentType) : 'bin'
|
|
135
|
-
|
|
136
|
-
const binary = await this[CLAIMS_KEY].fs.write(
|
|
137
|
-
`${uuidv7()}-http-request-activity-response-body.${ext}`,
|
|
138
|
-
response.body,
|
|
139
|
-
)
|
|
140
|
-
|
|
141
|
-
return this.toHttpOutput({
|
|
142
|
-
status: response.status,
|
|
143
|
-
headers: response.headers,
|
|
144
|
-
body: new Body(binary),
|
|
145
|
-
})
|
|
146
|
-
```
|
|
105
|
+
| `@t4h.framework/http` | The HTTP activity persists response bodies to `fs` and returns them wrapped in a `Body` — you read them with `res.body.json()` / `text()`. |
|
|
106
|
+
| `@t4h.framework/jsx` | Render activities write their output to `fs` and hand back a `Binary` you consume in the workflow. |
|
|
147
107
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
### JSX render activities (reference)
|
|
151
|
-
|
|
152
|
-
```typescript
|
|
153
|
-
public readonly claims = new Claim({
|
|
154
|
-
fs: FileSystemClaim,
|
|
155
|
-
})
|
|
156
|
-
|
|
157
|
-
public async toInput() {
|
|
158
|
-
return await this.claims.fs.write('rsc.txt', Buffer.from('Hello World!'))
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
public async run(binary: Binary) {
|
|
162
|
-
const stream = await binary.stream()
|
|
163
|
-
const chunks = await stream.toArray()
|
|
164
|
-
return Buffer.concat(chunks).toString()
|
|
165
|
-
}
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
Current `toInput` stubs are placeholders; real implementations should still use `FileSystemClaim.write(path, Buffer | Readable)` per README.
|
|
108
|
+
Orchestrate these package primitives from your workflow; the `fs` write/read happens inside them.
|
|
169
109
|
|
|
170
110
|
---
|
|
171
111
|
|
|
172
112
|
## Testing
|
|
173
113
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
From `AbstractHttpRequestActivity.spec.ts` — wrap activity execution in `Claim.run` and provide partial mocks:
|
|
114
|
+
Test the **workflow** with `TestWorkflowEnvironment` from
|
|
115
|
+
`@t4h.framework/core/testing`, mocking `FileSystemClaim` (plus whatever other
|
|
116
|
+
claims the triggered activities need, e.g. `HttpClientRequestClaim` for an HTTP
|
|
117
|
+
call). See the **framework-workflow-testing** skill for the harness API.
|
|
179
118
|
|
|
180
119
|
```typescript
|
|
181
|
-
import {
|
|
120
|
+
import { Readable } from 'node:stream'
|
|
121
|
+
import { Binary } from '@t4h.framework/core'
|
|
122
|
+
import { TestWorkflowEnvironment, mockClaim } from '@t4h.framework/core/testing'
|
|
182
123
|
import { FileSystemClaim } from '@t4h.framework/fs'
|
|
183
|
-
import { HttpClientRequestClaim } from '@t4h.framework/http'
|
|
184
124
|
|
|
185
|
-
|
|
125
|
+
class MockBinary extends Binary {
|
|
126
|
+
public readonly contentType: string
|
|
127
|
+
public readonly contentLength: number
|
|
128
|
+
|
|
129
|
+
constructor(private readonly data: Buffer, contentType: string) {
|
|
130
|
+
super()
|
|
131
|
+
this.contentType = contentType
|
|
132
|
+
this.contentLength = data.byteLength
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
public stream() {
|
|
136
|
+
return Readable.from(this.data)
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// mockImplementation: replay runs the workflow twice, so return a fresh Binary each call.
|
|
141
|
+
const mockWrite = vi.fn().mockImplementation(async (_path: string, readable: Readable) => {
|
|
186
142
|
const chunks = await readable.toArray()
|
|
187
|
-
return new MockBinary(Buffer.concat(chunks),
|
|
143
|
+
return new MockBinary(Buffer.concat(chunks), 'application/json')
|
|
188
144
|
})
|
|
189
145
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
{
|
|
197
|
-
provide: FileSystemClaim,
|
|
198
|
-
value: {
|
|
199
|
-
write: mockWrite,
|
|
200
|
-
read: vi.fn(),
|
|
201
|
-
} as unknown as FileSystemClaim,
|
|
202
|
-
},
|
|
146
|
+
const env = TestWorkflowEnvironment.create({ now: 0 })
|
|
147
|
+
|
|
148
|
+
await env.execute(workflow, input, {
|
|
149
|
+
claims: [
|
|
150
|
+
mockClaim(FileSystemClaim, { write: mockWrite, read: vi.fn() }),
|
|
151
|
+
// ...other claims the workflow's activities need
|
|
203
152
|
],
|
|
204
|
-
|
|
205
|
-
)
|
|
153
|
+
})
|
|
206
154
|
```
|
|
207
155
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
### Workflow-level tests
|
|
211
|
-
|
|
212
|
-
`WorkflowTesting.run` from `@t4h.framework/core/testing` runs the full claim stack when you pass explicit `FileSystemClaim` providers. For isolated activity unit tests, prefer `Claim.run` with partial mocks.
|
|
156
|
+
Assert `mockWrite` was called with the expected path and body when relevant.
|
|
213
157
|
|
|
214
158
|
---
|
|
215
159
|
|
|
@@ -218,7 +162,7 @@ Mock `write` to return a `Binary` compatible with what the activity expects (e.g
|
|
|
218
162
|
1. **Extend** `FileSystemClaim` from `@t4h.framework/fs`.
|
|
219
163
|
2. **Return** a `Binary` subclass (or core-compatible `Binary`) from `write` and `read` with correct `contentType`, `contentLength`, and `stream()`.
|
|
220
164
|
3. **Accept** `Readable | Buffer` in `write` (match the abstract signature).
|
|
221
|
-
4. **Register** `{ provide: FileSystemClaim, value: new YourImpl() }` in runtime
|
|
165
|
+
4. **Register** `{ provide: FileSystemClaim, value: new YourImpl() }` in the runtime (tests use `mockClaim(FileSystemClaim, ...)` instead).
|
|
222
166
|
5. **Keep paths deterministic** when activities replay — unpredictable temp paths are fine if only `Binary` metadata is serialized and replay re-executes `write`; know your serializer/replay story if `path` is persisted.
|
|
223
167
|
|
|
224
168
|
---
|
|
@@ -228,15 +172,13 @@ Mock `write` to return a `Binary` compatible with what the activity expects (e.g
|
|
|
228
172
|
1. Add `"@t4h.framework/fs": "workspace:^"` (or published version) to app `dependencies` when activities use `FileSystemClaim`.
|
|
229
173
|
2. Ensure **`@t4h.framework/core`** is available (peer of `fs`).
|
|
230
174
|
3. Register **`FileSystemClaim`** in the runtime or test `claims` array.
|
|
231
|
-
4.
|
|
232
|
-
5. Build before publish: `yarn build` in `packages/fs` (`tsc --project tsconfig.build.json`).
|
|
175
|
+
4. Orchestrate file I/O through the package activities that own `FileSystemClaim` (http, jsx) — workflow code never declares an activity or accesses `this.claims.fs`.
|
|
233
176
|
|
|
234
177
|
---
|
|
235
178
|
|
|
236
179
|
## Anti-patterns
|
|
237
180
|
|
|
238
181
|
- Importing from `@t4h.framework/fs` paths other than the package export (`FileSystemClaim` only).
|
|
239
|
-
- Calling `this.claims.fs` **outside** `Claim.run` / workflow execution context → `ClaimStorageNotActiveException`.
|
|
240
182
|
- Omitting `FileSystemClaim` from providers while using `@t4h.framework/http` or `@t4h.framework/jsx` activities that require `fs`.
|
|
241
183
|
- Assuming `packages/fs` ships a concrete disk/S3 implementation — it does not; implement and register `FileSystemClaim` in your runtime or test harness.
|
|
242
184
|
|
|
@@ -244,6 +186,6 @@ Mock `write` to return a `Binary` compatible with what the activity expects (e.g
|
|
|
244
186
|
|
|
245
187
|
## Related skills
|
|
246
188
|
|
|
247
|
-
- `framework-core` — `Claim`, `
|
|
189
|
+
- `framework-core` — `Claim`, `Binary`, and the `TestWorkflowEnvironment` testing harness.
|
|
248
190
|
- `framework-http` — HTTP activities that persist bodies via `FileSystemClaim`.
|
|
249
191
|
- `framework-jsx` — render activities that write/read via `fs`.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { AsyncActivityAwaitable, type Binary, StartWorkflowActivity, SyncActivity, Workflow } from '@t4h.framework/core';
|
|
2
|
+
import type Type from 'typebox';
|
|
3
|
+
import { type ParserFormat } from '../helpers/parsers.js';
|
|
4
|
+
export type FileSplitOptionsBase<TFormat extends ParserFormat> = {
|
|
5
|
+
format: TFormat;
|
|
6
|
+
encoding?: BufferEncoding;
|
|
7
|
+
};
|
|
8
|
+
type FileSplitOptionsLine = FileSplitOptionsBase<'txt' | 'ndjson'> & {
|
|
9
|
+
trimEnd?: boolean;
|
|
10
|
+
};
|
|
11
|
+
type FileSplitOptionsCSV = FileSplitOptionsBase<'csv'> & {
|
|
12
|
+
delimiter?: string;
|
|
13
|
+
};
|
|
14
|
+
type FileSplitOptionsCustom = FileSplitOptionsBase<string> & {
|
|
15
|
+
delimiter?: string;
|
|
16
|
+
trimEnd?: boolean;
|
|
17
|
+
};
|
|
18
|
+
type FileSplitOptions = FileSplitOptionsLine | FileSplitOptionsCSV | FileSplitOptionsCustom;
|
|
19
|
+
export type FileSplitActivityInput = {
|
|
20
|
+
binary: Binary;
|
|
21
|
+
options?: FileSplitOptions;
|
|
22
|
+
};
|
|
23
|
+
export declare class FileSplitActivity extends SyncActivity<readonly [binary: Binary, options?: FileSplitOptions], FileSplitActivityInput, Binary[], Binary[]> {
|
|
24
|
+
private readonly claims;
|
|
25
|
+
toInput(binary: Binary, options?: FileSplitOptions): FileSplitActivityInput;
|
|
26
|
+
run({ binary, options, }: FileSplitActivityInput): Promise<Binary[]>;
|
|
27
|
+
toOutput(binaries: Binary[]): Binary[];
|
|
28
|
+
}
|
|
29
|
+
type BinaryWorkflow = Workflow<Type.TObject<{
|
|
30
|
+
binary: Type.TSchema;
|
|
31
|
+
}>, any>;
|
|
32
|
+
export declare function split(binary: Binary, workflow: BinaryWorkflow, options?: FileSplitOptions & {
|
|
33
|
+
parallel?: boolean;
|
|
34
|
+
}): Promise<AsyncActivityAwaitable<StartWorkflowActivity>[]>;
|
|
35
|
+
export {};
|
|
36
|
+
//# sourceMappingURL=FileSplitActivity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileSplitActivity.d.ts","sourceRoot":"","sources":["../../src/activities/FileSplitActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,KAAK,MAAM,EAIX,qBAAqB,EACrB,YAAY,EACZ,QAAQ,EACT,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,IAAI,MAAM,SAAS,CAAA;AAG/B,OAAO,EAAW,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAElE,MAAM,MAAM,oBAAoB,CAAC,OAAO,SAAS,YAAY,IAAI;IAC/D,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,KAAK,oBAAoB,GAAG,oBAAoB,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG;IACnE,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB,CAAA;AAED,KAAK,mBAAmB,GAAG,oBAAoB,CAAC,KAAK,CAAC,GAAG;IACvD,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAGD,KAAK,sBAAsB,GAAG,oBAAoB,CAAC,MAAM,CAAC,GAAG;IAC3D,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB,CAAA;AAED,KAAK,gBAAgB,GACjB,oBAAoB,GACpB,mBAAmB,GACnB,sBAAsB,CAAA;AAE1B,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,gBAAgB,CAAA;CAC3B,CAAA;AAED,qBAAa,iBAAkB,SAAQ,YAAY,CACjD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,CAAC,EACrD,sBAAsB,EACtB,MAAM,EAAE,EACR,MAAM,EAAE,CACT;IACC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAErB;IAEK,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,gBAAgB,GACzB,sBAAsB;IAIZ,GAAG,CAAC,EACf,MAAM,EACN,OAA2B,GAC5B,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAqCtC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;CAG9C;AAED,KAAK,cAAc,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;IAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAA;CAAE,CAAC,EAAE,GAAG,CAAC,CAAA;AAE3E,wBAAsB,KAAK,CACzB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,cAAc,EACxB,OAAO,CAAC,EAAE,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,4DAgBpD"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { AsyncActivityAwaitable, Claim, History, startWorkflow, StartWorkflowActivity, SyncActivity, Workflow, } from '@t4h.framework/core';
|
|
2
|
+
import { FileSystemClaim } from '../FileSystemClaim.js';
|
|
3
|
+
import { parsers } from '../helpers/parsers.js';
|
|
4
|
+
export class FileSplitActivity extends SyncActivity {
|
|
5
|
+
claims = new Claim({
|
|
6
|
+
fs: FileSystemClaim,
|
|
7
|
+
});
|
|
8
|
+
toInput(binary, options) {
|
|
9
|
+
return { binary, options };
|
|
10
|
+
}
|
|
11
|
+
async run({ binary, options = { format: 'txt' }, }) {
|
|
12
|
+
const stream = await binary.stream();
|
|
13
|
+
const encoding = options.encoding ?? 'utf8';
|
|
14
|
+
stream.setEncoding(encoding);
|
|
15
|
+
const format = options.format;
|
|
16
|
+
const parser = parsers.get(format);
|
|
17
|
+
if (!parser)
|
|
18
|
+
throw new Error(`Unknown file split format: "${format}"`);
|
|
19
|
+
const delimiter = 'delimiter' in options ? options.delimiter : undefined;
|
|
20
|
+
const trimEnd = 'trimEnd' in options ? options.trimEnd : false;
|
|
21
|
+
const binaries = [];
|
|
22
|
+
let index = 0;
|
|
23
|
+
for await (const record of parser(stream, { encoding, delimiter })) {
|
|
24
|
+
const value = trimEnd ? record.trimEnd() : record;
|
|
25
|
+
if (!value)
|
|
26
|
+
continue;
|
|
27
|
+
binaries.push(await this.claims.fs.write(`lines/${index++}.txt`, Buffer.from(value, encoding)));
|
|
28
|
+
}
|
|
29
|
+
return binaries;
|
|
30
|
+
}
|
|
31
|
+
toOutput(binaries) {
|
|
32
|
+
return binaries;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export async function split(binary, workflow, options) {
|
|
36
|
+
const binaries = await History.reconciler(FileSplitActivity, binary, options);
|
|
37
|
+
const workflows = [];
|
|
38
|
+
for (const entry of binaries) {
|
|
39
|
+
workflows.push(await startWorkflow(workflow, {
|
|
40
|
+
binary: entry,
|
|
41
|
+
parallel: options?.parallel,
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
return workflows;
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=FileSplitActivity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileSplitActivity.js","sourceRoot":"","sources":["../../src/activities/FileSplitActivity.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EAEtB,KAAK,EACL,OAAO,EACP,aAAa,EACb,qBAAqB,EACrB,YAAY,EACZ,QAAQ,GACT,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,OAAO,EAAqB,MAAM,uBAAuB,CAAA;AA+BlE,MAAM,OAAO,iBAAkB,SAAQ,YAKtC;IACkB,MAAM,GAAG,IAAI,KAAK,CAAC;QAClC,EAAE,EAAE,eAAe;KACpB,CAAC,CAAA;IAEK,OAAO,CACZ,MAAc,EACd,OAA0B;QAE1B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,EACf,MAAM,EACN,OAAO,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GACJ;QACvB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAA;QAEpC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAA;QAE3C,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;QAE5B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAElC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,GAAG,CAAC,CAAA;QAEtE,MAAM,SAAS,GAAG,WAAW,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;QAExE,MAAM,OAAO,GAAG,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAA;QAE9D,MAAM,QAAQ,GAAa,EAAE,CAAA;QAE7B,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YACnE,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAA;YAEjD,IAAI,CAAC,KAAK;gBAAE,SAAQ;YAEpB,QAAQ,CAAC,IAAI,CACX,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CACxB,SAAS,KAAK,EAAE,MAAM,EACtB,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAC7B,CACF,CAAA;QACH,CAAC;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEM,QAAQ,CAAC,QAAkB;QAChC,OAAO,QAAQ,CAAA;IACjB,CAAC;CACF;AAID,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,MAAc,EACd,QAAwB,EACxB,OAAmD;IAEnD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IAE7E,MAAM,SAAS,GAAoD,EAAE,CAAA;IAErE,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,SAAS,CAAC,IAAI,CACZ,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IAED,OAAO,SAAS,CAAA;AAClB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileSplitActivity.spec.d.ts","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { Claim } from '@t4h.framework/core';
|
|
3
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
4
|
+
import { FileSystemClaim } from '../../FileSystemClaim.js';
|
|
5
|
+
import { FileSplitActivity, } from '../FileSplitActivity.js';
|
|
6
|
+
function makeBinary(content) {
|
|
7
|
+
const stream = Readable.from([content]);
|
|
8
|
+
const binary = {
|
|
9
|
+
contentType: 'text/plain',
|
|
10
|
+
contentLength: Buffer.byteLength(content),
|
|
11
|
+
stream: () => stream,
|
|
12
|
+
};
|
|
13
|
+
return { binary, stream };
|
|
14
|
+
}
|
|
15
|
+
async function runActivity(input) {
|
|
16
|
+
const write = vi.fn((path, content) => ({ path, content }));
|
|
17
|
+
const binaries = await Claim.run([
|
|
18
|
+
{
|
|
19
|
+
provide: FileSystemClaim,
|
|
20
|
+
value: { write },
|
|
21
|
+
},
|
|
22
|
+
], () => {
|
|
23
|
+
const activity = new FileSplitActivity();
|
|
24
|
+
return activity.run(input);
|
|
25
|
+
});
|
|
26
|
+
return { binaries, write };
|
|
27
|
+
}
|
|
28
|
+
describe('BinarySplitActivity', () => {
|
|
29
|
+
describe('toInput', () => {
|
|
30
|
+
it('wraps the binary and options into the activity input', () => {
|
|
31
|
+
const activity = new FileSplitActivity();
|
|
32
|
+
const { binary } = makeBinary('');
|
|
33
|
+
const options = { format: 'csv', delimiter: ';' };
|
|
34
|
+
const input = activity.toInput(binary, options);
|
|
35
|
+
expect(input).toEqual({ binary, options });
|
|
36
|
+
});
|
|
37
|
+
it('leaves options undefined when none are provided', () => {
|
|
38
|
+
const activity = new FileSplitActivity();
|
|
39
|
+
const { binary } = makeBinary('');
|
|
40
|
+
const input = activity.toInput(binary);
|
|
41
|
+
expect(input).toEqual({ binary, options: undefined });
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
describe('toOutput', () => {
|
|
45
|
+
it('returns the binaries unchanged', () => {
|
|
46
|
+
const activity = new FileSplitActivity();
|
|
47
|
+
const binaries = [{}, {}];
|
|
48
|
+
expect(activity.toOutput(binaries)).toBe(binaries);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
describe('run (txt / ndjson)', () => {
|
|
52
|
+
it('writes one file per non-empty line using an incrementing index', async () => {
|
|
53
|
+
const { binary } = makeBinary('foo\nbar\nbaz\n');
|
|
54
|
+
const { binaries, write } = await runActivity({ binary });
|
|
55
|
+
expect(write).toHaveBeenCalledTimes(3);
|
|
56
|
+
expect(write).toHaveBeenNthCalledWith(1, 'lines/0.txt', Buffer.from('foo', 'utf8'));
|
|
57
|
+
expect(write).toHaveBeenNthCalledWith(2, 'lines/1.txt', Buffer.from('bar', 'utf8'));
|
|
58
|
+
expect(write).toHaveBeenNthCalledWith(3, 'lines/2.txt', Buffer.from('baz', 'utf8'));
|
|
59
|
+
expect(binaries).toHaveLength(3);
|
|
60
|
+
});
|
|
61
|
+
it('defaults to the txt format when no options are provided', async () => {
|
|
62
|
+
const { binary } = makeBinary('only-line\n');
|
|
63
|
+
const { write } = await runActivity({ binary });
|
|
64
|
+
expect(write).toHaveBeenCalledOnce();
|
|
65
|
+
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from('only-line', 'utf8'));
|
|
66
|
+
});
|
|
67
|
+
it('treats ndjson identically to txt (line based)', async () => {
|
|
68
|
+
const { binary } = makeBinary('{"a":1}\n{"b":2}\n');
|
|
69
|
+
const { write } = await runActivity({
|
|
70
|
+
binary,
|
|
71
|
+
options: { format: 'ndjson' },
|
|
72
|
+
});
|
|
73
|
+
expect(write).toHaveBeenNthCalledWith(1, 'lines/0.txt', Buffer.from('{"a":1}', 'utf8'));
|
|
74
|
+
expect(write).toHaveBeenNthCalledWith(2, 'lines/1.txt', Buffer.from('{"b":2}', 'utf8'));
|
|
75
|
+
});
|
|
76
|
+
it('skips truly empty lines but keeps whitespace-only lines verbatim by default (trimEnd off)', async () => {
|
|
77
|
+
const { binary } = makeBinary('foo\n\n \nbar\n');
|
|
78
|
+
const { write } = await runActivity({ binary });
|
|
79
|
+
// With trimEnd off, only the empty line is dropped; ' ' is written as-is.
|
|
80
|
+
expect(write).toHaveBeenCalledTimes(3);
|
|
81
|
+
expect(write).toHaveBeenNthCalledWith(1, 'lines/0.txt', Buffer.from('foo', 'utf8'));
|
|
82
|
+
expect(write).toHaveBeenNthCalledWith(2, 'lines/1.txt', Buffer.from(' ', 'utf8'));
|
|
83
|
+
expect(write).toHaveBeenNthCalledWith(3, 'lines/2.txt', Buffer.from('bar', 'utf8'));
|
|
84
|
+
});
|
|
85
|
+
it('trims trailing whitespace and drops now-empty lines when trimEnd is enabled', async () => {
|
|
86
|
+
const { binary } = makeBinary(' keep-leading \n \n');
|
|
87
|
+
const { write } = await runActivity({
|
|
88
|
+
binary,
|
|
89
|
+
options: { format: 'txt', trimEnd: true },
|
|
90
|
+
});
|
|
91
|
+
// Leading whitespace is preserved; the whitespace-only line trims to '' and is skipped.
|
|
92
|
+
expect(write).toHaveBeenCalledOnce();
|
|
93
|
+
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from(' keep-leading', 'utf8'));
|
|
94
|
+
});
|
|
95
|
+
it('returns an empty array when the stream has no usable lines', async () => {
|
|
96
|
+
const { binary } = makeBinary('\n\n');
|
|
97
|
+
const { binaries, write } = await runActivity({ binary });
|
|
98
|
+
expect(binaries).toEqual([]);
|
|
99
|
+
expect(write).not.toHaveBeenCalled();
|
|
100
|
+
});
|
|
101
|
+
it('sets the default utf8 encoding on the stream', async () => {
|
|
102
|
+
const { binary, stream } = makeBinary('foo\n');
|
|
103
|
+
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
104
|
+
await runActivity({ binary });
|
|
105
|
+
expect(setEncoding).toHaveBeenCalledWith('utf8');
|
|
106
|
+
});
|
|
107
|
+
it('honors a custom encoding on both the stream and the written buffer', async () => {
|
|
108
|
+
const { binary, stream } = makeBinary('foo\n');
|
|
109
|
+
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
110
|
+
const { write } = await runActivity({
|
|
111
|
+
binary,
|
|
112
|
+
options: { format: 'txt', encoding: 'latin1' },
|
|
113
|
+
});
|
|
114
|
+
expect(setEncoding).toHaveBeenCalledWith('latin1');
|
|
115
|
+
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from('foo', 'latin1'));
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
describe('run (csv)', () => {
|
|
119
|
+
it('parses each row and writes it as a serialized JSON object', async () => {
|
|
120
|
+
const { binary } = makeBinary('name,age\nalice,30\nbob,25\n');
|
|
121
|
+
const { binaries, write } = await runActivity({
|
|
122
|
+
binary,
|
|
123
|
+
options: { format: 'csv' },
|
|
124
|
+
});
|
|
125
|
+
expect(write).toHaveBeenCalledTimes(2);
|
|
126
|
+
expect(write).toHaveBeenNthCalledWith(1, 'lines/0.txt', Buffer.from(JSON.stringify({ name: 'alice', age: '30' }), 'utf8'));
|
|
127
|
+
expect(write).toHaveBeenNthCalledWith(2, 'lines/1.txt', Buffer.from(JSON.stringify({ name: 'bob', age: '25' }), 'utf8'));
|
|
128
|
+
expect(binaries).toHaveLength(2);
|
|
129
|
+
});
|
|
130
|
+
it('honors a custom delimiter when parsing rows', async () => {
|
|
131
|
+
const { binary } = makeBinary('a;b\n1;2\n');
|
|
132
|
+
const { write } = await runActivity({
|
|
133
|
+
binary,
|
|
134
|
+
options: { format: 'csv', delimiter: ';' },
|
|
135
|
+
});
|
|
136
|
+
// A wrong delimiter would collapse the two columns into a single key.
|
|
137
|
+
expect(write).toHaveBeenCalledOnce();
|
|
138
|
+
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from(JSON.stringify({ a: '1', b: '2' }), 'utf8'));
|
|
139
|
+
});
|
|
140
|
+
it('writes nothing when the csv contains only a header row', async () => {
|
|
141
|
+
const { binary } = makeBinary('a,b\n');
|
|
142
|
+
const { binaries, write } = await runActivity({
|
|
143
|
+
binary,
|
|
144
|
+
options: { format: 'csv' },
|
|
145
|
+
});
|
|
146
|
+
expect(binaries).toEqual([]);
|
|
147
|
+
expect(write).not.toHaveBeenCalled();
|
|
148
|
+
});
|
|
149
|
+
it('applies a custom encoding to both the stream and the serialized buffer', async () => {
|
|
150
|
+
const { binary, stream } = makeBinary('a,b\n1,2\n');
|
|
151
|
+
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
152
|
+
const { write } = await runActivity({
|
|
153
|
+
binary,
|
|
154
|
+
options: { format: 'csv', encoding: 'latin1' },
|
|
155
|
+
});
|
|
156
|
+
expect(setEncoding).toHaveBeenCalledWith('latin1');
|
|
157
|
+
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from(JSON.stringify({ a: '1', b: '2' }), 'latin1'));
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
describe('run (claims)', () => {
|
|
161
|
+
it('throws when no FileSystemClaim provider is registered', async () => {
|
|
162
|
+
const { binary } = makeBinary('foo\n');
|
|
163
|
+
await expect(Claim.run([], () => new FileSplitActivity().run({ binary }))).rejects.toThrow(/FileSystemClaim/);
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
//# sourceMappingURL=FileSplitActivity.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileSplitActivity.spec.js","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAe,KAAK,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EACL,iBAAiB,GAElB,MAAM,yBAAyB,CAAA;AAEhC,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;IAEvC,MAAM,MAAM,GAAG;QACb,WAAW,EAAE,YAAY;QACzB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;QACzC,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM;KACA,CAAA;IAEtB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC3B,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAA6B;IAItD,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CACjB,CAAC,IAAY,EAAE,OAAe,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAsB,CAC5E,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAC9B;QACE;YACE,OAAO,EAAE,eAAe;YACxB,KAAK,EAAE,EAAE,KAAK,EAAgC;SAC/C;KACF,EACD,GAAG,EAAE;QACH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;QAExC,OAAO,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC5B,CAAC,CACF,CAAA;IAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAA;AAC5B,CAAC;AAED,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;YACjC,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAW,CAAA;YAE1D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAE/C,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;YAEjC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YAEtC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;QACvD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,QAAQ,GAAG,CAAC,EAAE,EAAE,EAAE,CAAwB,CAAA;YAEhD,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAA;YAEhD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YAEzD,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAC3B,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAC3B,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAC3B,CAAA;YACD,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAClC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,aAAa,CAAC,CAAA;YAE5C,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YAE/C,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,EAAE,CAAA;YACpC,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CACjC,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,oBAAoB,CAAC,CAAA;YAEnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAClC,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE;aAC9B,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAC/B,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAC/B,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2FAA2F,EAAE,KAAK,IAAI,EAAE;YACzG,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,mBAAmB,CAAC,CAAA;YAElD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YAE/C,4EAA4E;YAC5E,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAC3B,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAC3B,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAC3B,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;YAC3F,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,0BAA0B,CAAC,CAAA;YAEzD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAClC,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE;aAC1C,CAAC,CAAA;YAEF,wFAAwF;YACxF,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,EAAE,CAAA;YACpC,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,CACtC,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;YAErC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YAEzD,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC5B,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;QACtC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;YAC9C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;YAEnD,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YAE7B,MAAM,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA;QAClD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;YAClF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;YAC9C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;YAEnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAClC,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE;aAC/C,CAAC,CAAA;YAEF,MAAM,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAC7B,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,8BAA8B,CAAC,CAAA;YAE7D,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAC5C,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aAC3B,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,CAClE,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,CAChE,CAAA;YACD,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAClC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,YAAY,CAAC,CAAA;YAE3C,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAClC,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE;aAC3C,CAAC,CAAA;YAEF,sEAAsE;YACtE,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,EAAE,CAAA;YACpC,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,CACxD,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;YAEtC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAC5C,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aAC3B,CAAC,CAAA;YAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC5B,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;QACtC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;YACtF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,YAAY,CAAC,CAAA;YACnD,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;YAEnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC;gBAClC,MAAM;gBACN,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE;aAC/C,CAAC,CAAA;YAEF,MAAM,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,CAC1D,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;YAEtC,MAAM,MAAM,CACV,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,iBAAiB,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAC7D,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-reader.spec.d.ts","sourceRoot":"","sources":["../../../src/helpers/__tests__/csv-reader.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
3
|
+
import { csvReader } from '../csv-reader.js';
|
|
4
|
+
async function collect(iterable) {
|
|
5
|
+
const records = [];
|
|
6
|
+
for await (const record of iterable)
|
|
7
|
+
records.push(record);
|
|
8
|
+
return records;
|
|
9
|
+
}
|
|
10
|
+
describe('csvReader', () => {
|
|
11
|
+
it('parses rows into objects keyed by the header row', async () => {
|
|
12
|
+
const stream = Readable.from(['name,age\nalice,30\nbob,25\n']);
|
|
13
|
+
const records = await collect(csvReader(stream));
|
|
14
|
+
expect(records).toEqual([
|
|
15
|
+
JSON.stringify({ name: 'alice', age: '30' }),
|
|
16
|
+
JSON.stringify({ name: 'bob', age: '25' }),
|
|
17
|
+
]);
|
|
18
|
+
});
|
|
19
|
+
it('uses comma as the default delimiter', async () => {
|
|
20
|
+
const stream = Readable.from(['a,b\n1,2\n']);
|
|
21
|
+
const records = await collect(csvReader(stream));
|
|
22
|
+
expect(records).toEqual([JSON.stringify({ a: '1', b: '2' })]);
|
|
23
|
+
});
|
|
24
|
+
it('respects a custom delimiter', async () => {
|
|
25
|
+
const stream = Readable.from(['a;b\n1;2\n']);
|
|
26
|
+
const records = await collect(csvReader(stream, { delimiter: ';' }));
|
|
27
|
+
expect(records).toEqual([JSON.stringify({ a: '1', b: '2' })]);
|
|
28
|
+
});
|
|
29
|
+
it('tolerates rows with a mismatched column count (relax_column_count)', async () => {
|
|
30
|
+
const stream = Readable.from(['a,b\n1\n2,3,4\n']);
|
|
31
|
+
const records = await collect(csvReader(stream, { delimiter: ',' }));
|
|
32
|
+
expect(records).toEqual([
|
|
33
|
+
JSON.stringify({ a: '1' }),
|
|
34
|
+
JSON.stringify({ a: '2', b: '3' }),
|
|
35
|
+
]);
|
|
36
|
+
});
|
|
37
|
+
it('yields nothing for an empty stream', async () => {
|
|
38
|
+
const stream = Readable.from(['']);
|
|
39
|
+
const records = await collect(csvReader(stream));
|
|
40
|
+
expect(records).toEqual([]);
|
|
41
|
+
});
|
|
42
|
+
it('yields nothing when only the header row is present', async () => {
|
|
43
|
+
const stream = Readable.from(['a,b\n']);
|
|
44
|
+
const records = await collect(csvReader(stream));
|
|
45
|
+
expect(records).toEqual([]);
|
|
46
|
+
});
|
|
47
|
+
it('sets the stream encoding when the encoding option is provided', async () => {
|
|
48
|
+
const stream = Readable.from(['a,b\n1,2\n']);
|
|
49
|
+
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
50
|
+
await collect(csvReader(stream, { encoding: 'latin1' }));
|
|
51
|
+
expect(setEncoding).toHaveBeenCalledWith('latin1');
|
|
52
|
+
});
|
|
53
|
+
it('does not touch the stream encoding when no encoding option is provided', async () => {
|
|
54
|
+
const stream = Readable.from(['a,b\n1,2\n']);
|
|
55
|
+
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
56
|
+
await collect(csvReader(stream));
|
|
57
|
+
expect(setEncoding).not.toHaveBeenCalled();
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
//# sourceMappingURL=csv-reader.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-reader.spec.js","sourceRoot":"","sources":["../../../src/helpers/__tests__/csv-reader.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAE5C,KAAK,UAAU,OAAO,CAAI,QAA0B;IAClD,MAAM,OAAO,GAAQ,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEzD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAA;QAE9D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;SAC3C,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAA;QAEjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;SACnC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAElC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;QAEvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QAEnD,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAA;QAExD,MAAM,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QAEnD,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;IAC5C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream';
|
|
2
|
+
interface Options {
|
|
3
|
+
delimiter?: string;
|
|
4
|
+
encoding?: BufferEncoding;
|
|
5
|
+
}
|
|
6
|
+
export declare function csvReader(stream: Readable, options?: Options): AsyncGenerator<string, void, unknown>;
|
|
7
|
+
export {};
|
|
8
|
+
//# sourceMappingURL=csv-reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-reader.d.ts","sourceRoot":"","sources":["../../src/helpers/csv-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAI3C,UAAU,OAAO;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B;AAED,wBAAuB,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,yCAcnE"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { parse } from 'csv-parse';
|
|
2
|
+
export async function* csvReader(stream, options) {
|
|
3
|
+
if (options?.encoding)
|
|
4
|
+
stream.setEncoding(options.encoding);
|
|
5
|
+
const reader = stream.pipe(parse({
|
|
6
|
+
delimiter: options?.delimiter,
|
|
7
|
+
columns: true,
|
|
8
|
+
relax_column_count: true,
|
|
9
|
+
}));
|
|
10
|
+
for await (const record of reader) {
|
|
11
|
+
yield JSON.stringify(record);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=csv-reader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-reader.js","sourceRoot":"","sources":["../../src/helpers/csv-reader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AAOjC,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,MAAgB,EAAE,OAAiB;IAClE,IAAI,OAAO,EAAE,QAAQ;QAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAE3D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC;QACJ,SAAS,EAAE,OAAO,EAAE,SAAS;QAC7B,OAAO,EAAE,IAAI;QACb,kBAAkB,EAAE,IAAI;KACzB,CAAC,CACH,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream';
|
|
2
|
+
export type Parser = (stream: Readable, options: {
|
|
3
|
+
encoding: BufferEncoding;
|
|
4
|
+
delimiter?: string;
|
|
5
|
+
}) => AsyncIterable<string>;
|
|
6
|
+
export type ParserFormat = 'txt' | 'ndjson' | 'csv' | string;
|
|
7
|
+
export declare const parsers: Map<string, Parser>;
|
|
8
|
+
export declare function registerParser(name: ParserFormat, parser: Parser): void;
|
|
9
|
+
//# sourceMappingURL=parsers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/helpers/parsers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAI3C,MAAM,MAAM,MAAM,GAAG,CACnB,MAAM,EAAE,QAAQ,EAChB,OAAO,EAAE;IAAE,QAAQ,EAAE,cAAc,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KACtD,aAAa,CAAC,MAAM,CAAC,CAAA;AAE1B,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAA;AAE5D,eAAO,MAAM,OAAO,qBAQlB,CAAA;AAEF,wBAAgB,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAEvE"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { createInterface } from 'node:readline';
|
|
2
|
+
import { csvReader } from './csv-reader.js';
|
|
3
|
+
export const parsers = new Map([
|
|
4
|
+
['txt', stream => createInterface({ input: stream, crlfDelay: Infinity })],
|
|
5
|
+
['ndjson', stream => createInterface({ input: stream, crlfDelay: Infinity })],
|
|
6
|
+
[
|
|
7
|
+
'csv',
|
|
8
|
+
(stream, { encoding, delimiter }) => csvReader(stream, { encoding, delimiter }),
|
|
9
|
+
],
|
|
10
|
+
]);
|
|
11
|
+
export function registerParser(name, parser) {
|
|
12
|
+
parsers.set(name, parser);
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=parsers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/helpers/parsers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAG/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAS3C,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAuB;IACnD,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7E;QACE,KAAK;QACL,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,CAClC,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;KAC7C;CACF,CAAC,CAAA;AAEF,MAAM,UAAU,cAAc,CAAC,IAAkB,EAAE,MAAc;IAC/D,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAC3B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAmBvD,wBAAgB,mBAAmB,CACjC,cAAc,GAAE,OAAO,CAAC,eAAe,CAAM,gEAiB9C"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { Binary } from '@t4h.framework/core';
|
|
3
|
+
import { mockClaim } from '@t4h.framework/core/testing';
|
|
4
|
+
import { FileSystemClaim } from '../FileSystemClaim.js';
|
|
5
|
+
class MemoryBinary extends Binary {
|
|
6
|
+
data;
|
|
7
|
+
contentType;
|
|
8
|
+
constructor(data, contentType) {
|
|
9
|
+
super();
|
|
10
|
+
this.data = data;
|
|
11
|
+
this.contentType = contentType;
|
|
12
|
+
}
|
|
13
|
+
get contentLength() {
|
|
14
|
+
return this.data.byteLength;
|
|
15
|
+
}
|
|
16
|
+
stream() {
|
|
17
|
+
return Readable.from(this.data);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export function mockFileSystemClaim(implementation = {}) {
|
|
21
|
+
return mockClaim(FileSystemClaim, {
|
|
22
|
+
write: async (_filepath, readable) => {
|
|
23
|
+
const chunks = [];
|
|
24
|
+
for await (const chunk of readable)
|
|
25
|
+
chunks.push(Buffer.from(chunk));
|
|
26
|
+
return new MemoryBinary(Buffer.concat(chunks), 'application/octet-stream');
|
|
27
|
+
},
|
|
28
|
+
read: async (filepath) => new MemoryBinary(Buffer.from(filepath), 'application/octet-stream'),
|
|
29
|
+
...implementation,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AAEvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAEvD,MAAM,YAAa,SAAQ,MAAM;IAEZ;IACD;IAFlB,YACmB,IAAY,EACb,WAAmB;QAEnC,KAAK,EAAE,CAAA;QAHU,SAAI,GAAJ,IAAI,CAAQ;QACb,gBAAW,GAAX,WAAW,CAAQ;IAGrC,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAA;IAC7B,CAAC;IAEM,MAAM;QACX,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;CACF;AAED,MAAM,UAAU,mBAAmB,CACjC,iBAA2C,EAAE;IAE7C,OAAO,SAAS,CAAC,eAAe,EAAE;QAChC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;YACnC,MAAM,MAAM,GAAa,EAAE,CAAA;YAE3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ;gBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;YAEnE,OAAO,IAAI,YAAY,CACrB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EACrB,0BAA0B,CAC3B,CAAA;QACH,CAAC;QACD,IAAI,EAAE,KAAK,EAAC,QAAQ,EAAC,EAAE,CACrB,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;QACrE,GAAG,cAAc;KAClB,CAAC,CAAA;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@t4h.framework/fs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "File system module for the T4H Framework",
|
|
5
5
|
"homepage": "https://github.com/tech4humans-brasil/framework/tree/main/packages/fs",
|
|
6
6
|
"bugs": "https://github.com/tech4humans-brasil/framework/issues",
|
|
@@ -30,11 +30,11 @@
|
|
|
30
30
|
"prepublishOnly": "yarn build"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
|
-
"@t4h.framework/core": "^0.
|
|
33
|
+
"@t4h.framework/core": "^0.8.0",
|
|
34
34
|
"typescript": "^5.9.3"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"@t4h.framework/core": "^0.
|
|
37
|
+
"@t4h.framework/core": "^0.8.0"
|
|
38
38
|
},
|
|
39
39
|
"packageManager": "yarn@4.12.0",
|
|
40
40
|
"engines": {
|