@t4h.framework/fs 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.ai/skills/framework-fs/SKILL.md +72 -115
  2. package/.ai/skills/framework-fs-split/SKILL.md +255 -0
  3. package/README.md +184 -3
  4. package/dist/FileSystemClaim.d.ts +4 -0
  5. package/dist/FileSystemClaim.d.ts.map +1 -1
  6. package/dist/FileSystemClaim.js.map +1 -1
  7. package/dist/activities/FileJoinActivity.d.ts +39 -0
  8. package/dist/activities/FileJoinActivity.d.ts.map +1 -0
  9. package/dist/activities/FileJoinActivity.js +87 -0
  10. package/dist/activities/FileJoinActivity.js.map +1 -0
  11. package/dist/activities/FileSplitActivity.d.ts +38 -0
  12. package/dist/activities/FileSplitActivity.d.ts.map +1 -0
  13. package/dist/activities/FileSplitActivity.js +51 -0
  14. package/dist/activities/FileSplitActivity.js.map +1 -0
  15. package/dist/activities/__tests__/FileJoinActivity.spec.d.ts +2 -0
  16. package/dist/activities/__tests__/FileJoinActivity.spec.d.ts.map +1 -0
  17. package/dist/activities/__tests__/FileJoinActivity.spec.js +160 -0
  18. package/dist/activities/__tests__/FileJoinActivity.spec.js.map +1 -0
  19. package/dist/activities/__tests__/FileSplitActivity.spec.d.ts +2 -0
  20. package/dist/activities/__tests__/FileSplitActivity.spec.d.ts.map +1 -0
  21. package/dist/activities/__tests__/FileSplitActivity.spec.js +149 -0
  22. package/dist/activities/__tests__/FileSplitActivity.spec.js.map +1 -0
  23. package/dist/fs.d.ts +3 -0
  24. package/dist/fs.d.ts.map +1 -1
  25. package/dist/fs.js +3 -0
  26. package/dist/fs.js.map +1 -1
  27. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts +2 -0
  28. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts.map +1 -0
  29. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +127 -0
  30. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +1 -0
  31. package/dist/helpers/__tests__/csv-reader.spec.d.ts +2 -0
  32. package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +1 -0
  33. package/dist/helpers/__tests__/csv-reader.spec.js +60 -0
  34. package/dist/helpers/__tests__/csv-reader.spec.js.map +1 -0
  35. package/dist/helpers/byte-delimiter-scanner.d.ts +36 -0
  36. package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -0
  37. package/dist/helpers/byte-delimiter-scanner.js +61 -0
  38. package/dist/helpers/byte-delimiter-scanner.js.map +1 -0
  39. package/dist/helpers/csv-reader.d.ts +8 -0
  40. package/dist/helpers/csv-reader.d.ts.map +1 -0
  41. package/dist/helpers/csv-reader.js +14 -0
  42. package/dist/helpers/csv-reader.js.map +1 -0
  43. package/dist/helpers/parsers.d.ts +9 -0
  44. package/dist/helpers/parsers.d.ts.map +1 -0
  45. package/dist/helpers/parsers.js +14 -0
  46. package/dist/helpers/parsers.js.map +1 -0
  47. package/dist/testing/index.d.ts +3 -0
  48. package/dist/testing/index.d.ts.map +1 -0
  49. package/dist/testing/index.js +32 -0
  50. package/dist/testing/index.js.map +1 -0
  51. package/package.json +8 -5
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=FileSplitActivity.spec.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FileSplitActivity.spec.d.ts","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":""}
@@ -0,0 +1,149 @@
1
+ import { Readable } from 'node:stream';
2
+ import { buffer } from 'node:stream/consumers';
3
+ import { AsyncActivityResultFulfilled, AsyncActivityResultPending, Claim, WorkflowBatchRef, WorkflowClaim, } from '@t4h.framework/core';
4
+ import { describe, expect, it, vi } from 'vitest';
5
+ import { FileSystemClaim } from '../../FileSystemClaim.js';
6
+ import { FileSplitActivity, Split, } from '../FileSplitActivity.js';
7
+ function makeBinary(content) {
8
+ return {
9
+ contentType: 'text/plain',
10
+ contentLength: Buffer.byteLength(content),
11
+ stream: () => Readable.from([Buffer.from(content)]),
12
+ };
13
+ }
14
+ const childWorkflow = { id: 'child-workflow' };
15
+ async function bootstrapActivity(input) {
16
+ const durable = { id: 'durable-copy' };
17
+ const written = [];
18
+ const write = vi.fn(async (_path, readable) => {
19
+ written.push(await buffer(readable));
20
+ return durable;
21
+ });
22
+ const slice = vi.fn((binary, range, encoding) => ({ binary, range, encoding }));
23
+ const startMany = vi.fn(async (_workflow, inputs) => new WorkflowBatchRef('batch-1', inputs.length));
24
+ const result = await Claim.run([
25
+ {
26
+ provide: FileSystemClaim,
27
+ value: { write, slice },
28
+ },
29
+ {
30
+ provide: WorkflowClaim,
31
+ value: { startMany },
32
+ },
33
+ ], () => {
34
+ const activity = new FileSplitActivity();
35
+ activity.id = 'activity-1';
36
+ return activity.bootstrap(input);
37
+ });
38
+ return { result, durable, write, slice, startMany, written };
39
+ }
40
+ describe('FileSplitActivity', () => {
41
+ describe('toInput', () => {
42
+ it('stores the workflow by id alongside the binary and options', () => {
43
+ const activity = new FileSplitActivity();
44
+ const binary = makeBinary('');
45
+ const options = { delimiter: Split.line() };
46
+ expect(activity.toInput(binary, childWorkflow, options)).toEqual({
47
+ binary,
48
+ workflow: 'child-workflow',
49
+ options,
50
+ });
51
+ });
52
+ it('throws when the workflow has no id', () => {
53
+ const activity = new FileSplitActivity();
54
+ expect(() => activity.toInput(makeBinary(''), {}, {
55
+ delimiter: Split.line(),
56
+ })).toThrow(/without an id/);
57
+ });
58
+ });
59
+ describe('bootstrap', () => {
60
+ it('persists the source once, slices by ranges, starts the batch and suspends', async () => {
61
+ const { result, durable, write, slice, startMany, written } = await bootstrapActivity({
62
+ binary: makeBinary('foo\nbar\nbaz'),
63
+ workflow: 'child-workflow',
64
+ options: { delimiter: Split.line() },
65
+ });
66
+ expect(write).toHaveBeenCalledExactlyOnceWith('split/activity-1', expect.anything());
67
+ expect(written[0]?.toString()).toBe('foo\nbar\nbaz');
68
+ expect(slice).toHaveBeenCalledTimes(3);
69
+ expect(slice).toHaveBeenNthCalledWith(1, durable, { start: 0, end: 3 }, 'utf8');
70
+ expect(slice).toHaveBeenNthCalledWith(2, durable, { start: 4, end: 7 }, 'utf8');
71
+ expect(slice).toHaveBeenNthCalledWith(3, durable, { start: 8, end: 11 }, 'utf8');
72
+ expect(startMany).toHaveBeenCalledExactlyOnceWith('child-workflow', [
73
+ slice.mock.results[0]?.value,
74
+ slice.mock.results[1]?.value,
75
+ slice.mock.results[2]?.value,
76
+ ]);
77
+ expect(result).toBeInstanceOf(AsyncActivityResultPending);
78
+ expect(result.schema).toBeDefined();
79
+ });
80
+ it('splits positional records with the record-start delimiter', async () => {
81
+ const { slice, result } = await bootstrapActivity({
82
+ binary: makeBinary('10A;x\n10B;y\n10C;z'),
83
+ workflow: 'child-workflow',
84
+ options: { delimiter: Split.char('\n10', { next: 2 }) },
85
+ });
86
+ expect(slice.mock.calls.map(([, range]) => range)).toEqual([
87
+ { start: 0, end: 5 },
88
+ { start: 6, end: 11 },
89
+ { start: 12, end: 17 },
90
+ ]);
91
+ expect(result).toBeInstanceOf(AsyncActivityResultPending);
92
+ });
93
+ it('resolves immediately for a file with no usable slices', async () => {
94
+ const { result, slice, startMany } = await bootstrapActivity({
95
+ binary: makeBinary('\n\n'),
96
+ workflow: 'child-workflow',
97
+ options: { delimiter: Split.line() },
98
+ });
99
+ expect(slice).not.toHaveBeenCalled();
100
+ expect(startMany).toHaveBeenCalledExactlyOnceWith('child-workflow', []);
101
+ expect(result).toEqual(new AsyncActivityResultFulfilled({ batch: 'batch-1', count: 0 }));
102
+ });
103
+ it('schedules the timeout when awaiting the batch', async () => {
104
+ const before = Date.now();
105
+ const { result } = await bootstrapActivity({
106
+ binary: makeBinary('foo\nbar'),
107
+ workflow: 'child-workflow',
108
+ options: { delimiter: Split.line(), timeoutInMs: 60_000 },
109
+ });
110
+ const pending = result;
111
+ expect(pending).toBeInstanceOf(AsyncActivityResultPending);
112
+ expect(pending.timeoutAt?.getTime()).toBeGreaterThanOrEqual(before + 60_000);
113
+ expect(pending.timeoutAt?.getTime()).toBeLessThanOrEqual(Date.now() + 60_000);
114
+ });
115
+ it('suspends without a timeout when none is configured', async () => {
116
+ const { result } = await bootstrapActivity({
117
+ binary: makeBinary('foo\nbar'),
118
+ workflow: 'child-workflow',
119
+ options: { delimiter: Split.line() },
120
+ });
121
+ expect(result.timeoutAt).toBeUndefined();
122
+ });
123
+ it('forwards the configured encoding to the scanner and the slices', async () => {
124
+ const { slice } = await bootstrapActivity({
125
+ binary: makeBinary('a;b'),
126
+ workflow: 'child-workflow',
127
+ options: { delimiter: Split.char(';'), encoding: 'latin1' },
128
+ });
129
+ expect(slice).toHaveBeenCalledWith(expect.anything(), { start: 0, end: 1 }, 'latin1');
130
+ });
131
+ });
132
+ describe('advance', () => {
133
+ it('resolves with the batch reference payload once the batch is terminal', () => {
134
+ const activity = new FileSplitActivity();
135
+ const payload = { batch: 'batch-1', count: 3 };
136
+ expect(activity.advance(payload)).toEqual(new AsyncActivityResultFulfilled(payload));
137
+ });
138
+ });
139
+ describe('toOutput', () => {
140
+ it('maps the payload to a WorkflowBatchRef', () => {
141
+ const activity = new FileSplitActivity();
142
+ const result = activity.toOutput({ batch: 'batch-1', count: 7 });
143
+ expect(result).toBeInstanceOf(WorkflowBatchRef);
144
+ expect(result.id).toBe('batch-1');
145
+ expect(result.count).toBe(7);
146
+ });
147
+ });
148
+ });
149
+ //# sourceMappingURL=FileSplitActivity.spec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FileSplitActivity.spec.js","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EACL,4BAA4B,EAC5B,0BAA0B,EAE1B,KAAK,EAEL,gBAAgB,EAChB,aAAa,GACd,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EACL,iBAAiB,EAEjB,KAAK,GACN,MAAM,yBAAyB,CAAA;AAEhC,SAAS,UAAU,CAAC,OAAe;IACjC,OAAO;QACL,WAAW,EAAE,YAAY;QACzB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;QACzC,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAC/B,CAAA;AACxB,CAAC;AAED,MAAM,aAAa,GAAG,EAAE,EAAE,EAAE,gBAAgB,EAAmC,CAAA;AAE/E,KAAK,UAAU,iBAAiB,CAAC,KAA6B;IAC5D,MAAM,OAAO,GAAG,EAAE,EAAE,EAAE,cAAc,EAAuB,CAAA;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,KAAa,EAAE,QAAkB,EAAE,EAAE;QAC9D,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEpC,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CACjB,CAAC,MAAc,EAAE,KAAqC,EAAE,QAAQ,EAAE,EAAE,CAClE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAsB,CACrD,CAAA;IAED,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,CACrB,KAAK,EAAE,SAAS,EAAE,MAAsB,EAAE,EAAE,CAC1C,IAAI,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CACjD,CAAA;IAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAC5B;QACE;YACE,OAAO,EAAE,eAAe;YACxB,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAgC;SACtD;QACD;YACE,OAAO,EAAE,aAAa;YACtB,KAAK,EAAE,EAAE,SAAS,EAA8B;SACjD;KACF,EACD,GAAG,EAAE;QACH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;QAExC,QAAQ,CAAC,EAAE,GAAG,YAAY,CAAA;QAE1B,OAAO,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC,CACF,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;AAC9D,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAA;YAE3C,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC/D,MAAM;gBACN,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO;aACR,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YAExC,MAAM,CAAC,GAAG,EAAE,CACV,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,EAAwB,EAAE;gBACzD,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE;aACxB,CAAC,CACH,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QAC5B,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GACzD,MAAM,iBAAiB,CAAC;gBACtB,MAAM,EAAE,UAAU,CAAC,eAAe,CAAC;gBACnC,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEJ,MAAM,CAAC,KAAK,CAAC,CAAC,+BAA+B,CAC3C,kBAAkB,EAClB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAEpD,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,MAAM,CACP,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,MAAM,CACP,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,EACrB,MAAM,CACP,CAAA;YAED,MAAM,CAAC,SAAS,CAAC,CAAC,+BAA+B,CAAC,gBAAgB,EAAE;gBAClE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;aAC7B,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;YACzD,MAAM,CAAE,MAAqC,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACrE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBAChD,MAAM,EAAE,UAAU,CAAC,qBAAqB,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE;aACxD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;gBACzD,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;gBACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;gBACrB,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACvB,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBAC3D,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC;gBAC1B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,+BAA+B,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CACpB,IAAI,4BAA4B,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACjE,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAEzB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACzC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC;gBAC9B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE;aAC1D,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,MAAoC,CAAA;YAEpD,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;YAC1D,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CACzD,MAAM,GAAG,MAAM,CAChB,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,mBAAmB,CACtD,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CACpB,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACzC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC;gBAC9B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEF,MAAM,CAAE,MAAqC,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAA;QAC1E,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACxC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC;gBACzB,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE;aAC5D,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,MAAM,CAAC,QAAQ,EAAE,EACjB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,QAAQ,CACT,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;YAC9E,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YAE9C,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACvC,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAC1C,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
package/dist/fs.d.ts CHANGED
@@ -1,2 +1,5 @@
1
1
  export * from './FileSystemClaim.js';
2
+ export * from './activities/FileJoinActivity.js';
3
+ export * from './activities/FileSplitActivity.js';
4
+ export * from './helpers/byte-delimiter-scanner.js';
2
5
  //# sourceMappingURL=fs.d.ts.map
package/dist/fs.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA"}
1
+ {"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA;AACpC,cAAc,kCAAkC,CAAA;AAChD,cAAc,mCAAmC,CAAA;AACjD,cAAc,qCAAqC,CAAA"}
package/dist/fs.js CHANGED
@@ -1,2 +1,5 @@
1
1
  export * from './FileSystemClaim.js';
2
+ export * from './activities/FileJoinActivity.js';
3
+ export * from './activities/FileSplitActivity.js';
4
+ export * from './helpers/byte-delimiter-scanner.js';
2
5
  //# sourceMappingURL=fs.js.map
package/dist/fs.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA"}
1
+ {"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA;AACpC,cAAc,kCAAkC,CAAA;AAChD,cAAc,mCAAmC,CAAA;AACjD,cAAc,qCAAqC,CAAA"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=byte-delimiter-scanner.spec.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"byte-delimiter-scanner.spec.d.ts","sourceRoot":"","sources":["../../../src/helpers/__tests__/byte-delimiter-scanner.spec.ts"],"names":[],"mappings":""}
@@ -0,0 +1,127 @@
1
+ import { Readable } from 'node:stream';
2
+ import { buffer } from 'node:stream/consumers';
3
+ import { describe, expect, it } from 'vitest';
4
+ import { ByteDelimiterScanner, } from '../byte-delimiter-scanner.js';
5
+ async function scan(chunks, options) {
6
+ const scanner = new ByteDelimiterScanner(options);
7
+ const output = await buffer(Readable.from(chunks.map(chunk => Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))).pipe(scanner));
8
+ return { ranges: scanner.ranges, output };
9
+ }
10
+ describe('ByteDelimiterScanner', () => {
11
+ it('cuts at every delimiter, discarding it by default', async () => {
12
+ const { ranges } = await scan(['foo\nbar\nbaz'], {
13
+ delimiter: { value: '\n' },
14
+ });
15
+ expect(ranges).toEqual([
16
+ { start: 0, end: 3 },
17
+ { start: 4, end: 7 },
18
+ { start: 8, end: 11 },
19
+ ]);
20
+ });
21
+ it('skips the empty final slice when the file ends with a delimiter', async () => {
22
+ const { ranges } = await scan(['foo\nbar\n'], {
23
+ delimiter: { value: '\n' },
24
+ });
25
+ expect(ranges).toEqual([
26
+ { start: 0, end: 3 },
27
+ { start: 4, end: 7 },
28
+ ]);
29
+ });
30
+ it('keeps empty slices when skipEmpty is off', async () => {
31
+ const { ranges } = await scan(['foo\n\nbar'], {
32
+ delimiter: { value: '\n' },
33
+ skipEmpty: false,
34
+ });
35
+ expect(ranges).toEqual([
36
+ { start: 0, end: 3 },
37
+ { start: 4, end: 4 },
38
+ { start: 5, end: 8 },
39
+ ]);
40
+ });
41
+ it('passes the bytes through unchanged', async () => {
42
+ const content = 'foo\nbar\nbaz\n';
43
+ const { output } = await scan([content], { delimiter: { value: '\n' } });
44
+ expect(output.toString()).toBe(content);
45
+ });
46
+ it('finds a delimiter split across chunk boundaries', async () => {
47
+ const { ranges } = await scan(['foo\r', '\nbar'], {
48
+ delimiter: { value: '\r\n' },
49
+ });
50
+ expect(ranges).toEqual([
51
+ { start: 0, end: 3 },
52
+ { start: 5, end: 8 },
53
+ ]);
54
+ });
55
+ it('does not double count a match ending inside the kept tail', async () => {
56
+ // '\n' is both a full match and a possible prefix of the 2-byte tail kept
57
+ // between chunks; a rescan would produce a phantom empty slice.
58
+ const { ranges } = await scan(['a\n10b', '\n10c'], {
59
+ delimiter: { value: '\n10', keep: { next: 2 } },
60
+ });
61
+ expect(ranges).toEqual([
62
+ { start: 0, end: 1 },
63
+ { start: 2, end: 5 },
64
+ { start: 6, end: 9 },
65
+ ]);
66
+ });
67
+ it('supports the positional record-start cut: newline dies, marker stays with the next slice', async () => {
68
+ const content = '10A;x\n10B;y\n10C;z';
69
+ const { ranges } = await scan([content], {
70
+ delimiter: { value: '\n10', keep: { next: 2 } },
71
+ });
72
+ expect(ranges).toEqual([
73
+ { start: 0, end: 5 },
74
+ { start: 6, end: 11 },
75
+ { start: 12, end: 17 },
76
+ ]);
77
+ expect(content.slice(6, 11)).toBe('10B;y');
78
+ });
79
+ it('supports keeping delimiter bytes with the previous slice', async () => {
80
+ const { ranges } = await scan(['a;b;c'], {
81
+ delimiter: { value: ';', keep: { previous: 1 } },
82
+ });
83
+ expect(ranges).toEqual([
84
+ { start: 0, end: 2 },
85
+ { start: 2, end: 4 },
86
+ { start: 4, end: 5 },
87
+ ]);
88
+ });
89
+ it('cuts only every N occurrences to group records', async () => {
90
+ const { ranges } = await scan(['a\nb\nc\nd'], {
91
+ delimiter: { value: '\n' },
92
+ every: 2,
93
+ });
94
+ expect(ranges).toEqual([
95
+ { start: 0, end: 3 },
96
+ { start: 4, end: 7 },
97
+ ]);
98
+ });
99
+ it('yields the whole file as one slice when the delimiter never occurs', async () => {
100
+ const { ranges } = await scan(['no-delimiter-here'], {
101
+ delimiter: { value: '\n' },
102
+ });
103
+ expect(ranges).toEqual([{ start: 0, end: 17 }]);
104
+ });
105
+ it('converts the delimiter to bytes using the configured encoding', async () => {
106
+ const { ranges } = await scan([Buffer.from('aébé', 'latin1')], {
107
+ delimiter: { value: 'é' },
108
+ encoding: 'latin1',
109
+ });
110
+ expect(ranges).toEqual([
111
+ { start: 0, end: 1 },
112
+ { start: 2, end: 3 },
113
+ ]);
114
+ });
115
+ it('rejects an empty delimiter', () => {
116
+ expect(() => new ByteDelimiterScanner({ delimiter: { value: '' } })).toThrow(/must not be empty/);
117
+ });
118
+ it('rejects keep sizes exceeding the delimiter length', () => {
119
+ expect(() => new ByteDelimiterScanner({
120
+ delimiter: { value: ';', keep: { previous: 1, next: 1 } },
121
+ })).toThrow(/must not exceed/);
122
+ });
123
+ it('rejects a non-positive every', () => {
124
+ expect(() => new ByteDelimiterScanner({ delimiter: { value: ';' }, every: 0 })).toThrow(/positive integer/);
125
+ });
126
+ });
127
+ //# sourceMappingURL=byte-delimiter-scanner.spec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"byte-delimiter-scanner.spec.js","sourceRoot":"","sources":["../../../src/helpers/__tests__/byte-delimiter-scanner.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EACL,oBAAoB,GAGrB,MAAM,8BAA8B,CAAA;AAErC,KAAK,UAAU,IAAI,CACjB,MAA2B,EAC3B,OAAoC;IAEpC,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAA;IAEjD,MAAM,MAAM,GAAG,MAAM,MAAM,CACzB,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACjB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CACpD,CACF,CAAC,IAAI,CAAC,OAAO,CAAC,CAChB,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAC3C,CAAC;AAED,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,eAAe,CAAC,EAAE;YAC/C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;SACtB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1B,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,OAAO,GAAG,iBAAiB,CAAA;QAEjC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAExE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;YAChD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;SAC7B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,0EAA0E;QAC1E,gEAAgE;QAChE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE;YACjD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAChD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0FAA0F,EAAE,KAAK,IAAI,EAAE;QACxG,MAAM,OAAO,GAAG,qBAAqB,CAAA;QAErC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE;YACvC,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAChD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;YACrB,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;SACvB,CAAC,CAAA;QAEF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE;YACvC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;SACjD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1B,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,mBAAmB,CAAC,EAAE;YACnD,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE;YAC7D,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE;YACzB,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CACJ,GAAG,EAAE,CAAC,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAC7D,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,oBAAoB,CAAC;YACvB,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAC1D,CAAC,CACL,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CACJ,GAAG,EAAE,CAAC,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACxE,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=csv-reader.spec.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-reader.spec.d.ts","sourceRoot":"","sources":["../../../src/helpers/__tests__/csv-reader.spec.ts"],"names":[],"mappings":""}
@@ -0,0 +1,60 @@
1
+ import { Readable } from 'node:stream';
2
+ import { describe, expect, it, vi } from 'vitest';
3
+ import { csvReader } from '../csv-reader.js';
4
+ async function collect(iterable) {
5
+ const records = [];
6
+ for await (const record of iterable)
7
+ records.push(record);
8
+ return records;
9
+ }
10
+ describe('csvReader', () => {
11
+ it('parses rows into objects keyed by the header row', async () => {
12
+ const stream = Readable.from(['name,age\nalice,30\nbob,25\n']);
13
+ const records = await collect(csvReader(stream));
14
+ expect(records).toEqual([
15
+ JSON.stringify({ name: 'alice', age: '30' }),
16
+ JSON.stringify({ name: 'bob', age: '25' }),
17
+ ]);
18
+ });
19
+ it('uses comma as the default delimiter', async () => {
20
+ const stream = Readable.from(['a,b\n1,2\n']);
21
+ const records = await collect(csvReader(stream));
22
+ expect(records).toEqual([JSON.stringify({ a: '1', b: '2' })]);
23
+ });
24
+ it('respects a custom delimiter', async () => {
25
+ const stream = Readable.from(['a;b\n1;2\n']);
26
+ const records = await collect(csvReader(stream, { delimiter: ';' }));
27
+ expect(records).toEqual([JSON.stringify({ a: '1', b: '2' })]);
28
+ });
29
+ it('tolerates rows with a mismatched column count (relax_column_count)', async () => {
30
+ const stream = Readable.from(['a,b\n1\n2,3,4\n']);
31
+ const records = await collect(csvReader(stream, { delimiter: ',' }));
32
+ expect(records).toEqual([
33
+ JSON.stringify({ a: '1' }),
34
+ JSON.stringify({ a: '2', b: '3' }),
35
+ ]);
36
+ });
37
+ it('yields nothing for an empty stream', async () => {
38
+ const stream = Readable.from(['']);
39
+ const records = await collect(csvReader(stream));
40
+ expect(records).toEqual([]);
41
+ });
42
+ it('yields nothing when only the header row is present', async () => {
43
+ const stream = Readable.from(['a,b\n']);
44
+ const records = await collect(csvReader(stream));
45
+ expect(records).toEqual([]);
46
+ });
47
+ it('sets the stream encoding when the encoding option is provided', async () => {
48
+ const stream = Readable.from(['a,b\n1,2\n']);
49
+ const setEncoding = vi.spyOn(stream, 'setEncoding');
50
+ await collect(csvReader(stream, { encoding: 'latin1' }));
51
+ expect(setEncoding).toHaveBeenCalledWith('latin1');
52
+ });
53
+ it('does not touch the stream encoding when no encoding option is provided', async () => {
54
+ const stream = Readable.from(['a,b\n1,2\n']);
55
+ const setEncoding = vi.spyOn(stream, 'setEncoding');
56
+ await collect(csvReader(stream));
57
+ expect(setEncoding).not.toHaveBeenCalled();
58
+ });
59
+ });
60
+ //# sourceMappingURL=csv-reader.spec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-reader.spec.js","sourceRoot":"","sources":["../../../src/helpers/__tests__/csv-reader.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAE5C,KAAK,UAAU,OAAO,CAAI,QAA0B;IAClD,MAAM,OAAO,GAAQ,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEzD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAA;QAE9D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;SAC3C,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAA;QAEjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;SACnC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAElC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;QAEvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QAEnD,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAA;QAExD,MAAM,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QAEnD,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;IAC5C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -0,0 +1,36 @@
1
+ import { Transform, type TransformCallback } from 'node:stream';
2
+ export type ByteRange = {
3
+ start: number;
4
+ end: number;
5
+ };
6
+ export type FileSplitDelimiter = {
7
+ value: string;
8
+ keep?: {
9
+ previous?: number;
10
+ next?: number;
11
+ };
12
+ };
13
+ export type ByteDelimiterScannerOptions = {
14
+ delimiter: FileSplitDelimiter;
15
+ every?: number;
16
+ skipEmpty?: boolean;
17
+ encoding?: BufferEncoding;
18
+ };
19
+ export declare class ByteDelimiterScanner extends Transform {
20
+ readonly ranges: ByteRange[];
21
+ private readonly delimiter;
22
+ private readonly keepPrevious;
23
+ private readonly keepNext;
24
+ private readonly cutEvery;
25
+ private readonly skipEmpty;
26
+ private pending;
27
+ private pendingAt;
28
+ private sliceStart;
29
+ private matches;
30
+ constructor({ delimiter, every, skipEmpty, encoding, }: ByteDelimiterScannerOptions);
31
+ _transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback): void;
32
+ _flush(callback: TransformCallback): void;
33
+ private scan;
34
+ private cut;
35
+ }
36
+ //# sourceMappingURL=byte-delimiter-scanner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"byte-delimiter-scanner.d.ts","sourceRoot":"","sources":["../../src/helpers/byte-delimiter-scanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,IAAI,CAAC,EAAE,MAAM,CAAA;KACd,CAAA;CACF,CAAA;AAED,MAAM,MAAM,2BAA2B,GAAG;IACxC,SAAS,EAAE,kBAAkB,CAAA;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,qBAAa,oBAAqB,SAAQ,SAAS;IACjD,SAAgB,MAAM,EAAE,SAAS,EAAE,CAAK;IAExC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,OAAO,CAAI;gBAEP,EACV,SAAS,EACT,KAAS,EACT,SAAgB,EAChB,QAAiB,GAClB,EAAE,2BAA2B;IAwBvB,UAAU,CACf,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,iBAAiB;IAStB,MAAM,CAAC,QAAQ,EAAE,iBAAiB;IAMzC,OAAO,CAAC,IAAI;IA8BZ,OAAO,CAAC,GAAG;CAKZ"}
@@ -0,0 +1,61 @@
1
+ import { Transform } from 'node:stream';
2
+ export class ByteDelimiterScanner extends Transform {
3
+ ranges = [];
4
+ delimiter;
5
+ keepPrevious;
6
+ keepNext;
7
+ cutEvery;
8
+ skipEmpty;
9
+ pending = Buffer.alloc(0);
10
+ pendingAt = 0;
11
+ sliceStart = 0;
12
+ matches = 0;
13
+ constructor({ delimiter, every = 1, skipEmpty = true, encoding = 'utf8', }) {
14
+ super();
15
+ this.delimiter = Buffer.from(delimiter.value, encoding);
16
+ this.keepPrevious = delimiter.keep?.previous ?? 0;
17
+ this.keepNext = delimiter.keep?.next ?? 0;
18
+ this.cutEvery = every;
19
+ this.skipEmpty = skipEmpty;
20
+ if (this.delimiter.length === 0)
21
+ throw new TypeError('Split delimiter must not be empty');
22
+ if (this.keepPrevious < 0 || this.keepNext < 0)
23
+ throw new TypeError('Split delimiter keep sizes must not be negative');
24
+ if (this.keepPrevious + this.keepNext > this.delimiter.length)
25
+ throw new TypeError('Split delimiter keep sizes must not exceed the delimiter length');
26
+ if (!Number.isInteger(every) || every < 1)
27
+ throw new TypeError('Split every must be a positive integer');
28
+ }
29
+ _transform(chunk, _encoding, callback) {
30
+ this.scan(this.pending.length > 0 ? Buffer.concat([this.pending, chunk]) : chunk);
31
+ callback(null, chunk);
32
+ }
33
+ _flush(callback) {
34
+ this.cut(this.sliceStart, this.pendingAt + this.pending.length);
35
+ callback();
36
+ }
37
+ scan(buffer) {
38
+ let searchFrom = 0;
39
+ while (true) {
40
+ const index = buffer.indexOf(this.delimiter, searchFrom);
41
+ if (index === -1)
42
+ break;
43
+ const at = this.pendingAt + index;
44
+ searchFrom = index + this.delimiter.length;
45
+ this.matches += 1;
46
+ if (this.matches % this.cutEvery !== 0)
47
+ continue;
48
+ this.cut(this.sliceStart, at + this.keepPrevious);
49
+ this.sliceStart = at + this.delimiter.length - this.keepNext;
50
+ }
51
+ const tail = Math.max(searchFrom, buffer.length - (this.delimiter.length - 1));
52
+ this.pending = buffer.subarray(tail);
53
+ this.pendingAt += tail;
54
+ }
55
+ cut(start, end) {
56
+ if (this.skipEmpty && end <= start)
57
+ return;
58
+ this.ranges.push({ start, end });
59
+ }
60
+ }
61
+ //# sourceMappingURL=byte-delimiter-scanner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"byte-delimiter-scanner.js","sourceRoot":"","sources":["../../src/helpers/byte-delimiter-scanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAA;AAsB/D,MAAM,OAAO,oBAAqB,SAAQ,SAAS;IACjC,MAAM,GAAgB,EAAE,CAAA;IAEvB,SAAS,CAAQ;IACjB,YAAY,CAAQ;IACpB,QAAQ,CAAQ;IAChB,QAAQ,CAAQ;IAChB,SAAS,CAAS;IAE3B,OAAO,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACjC,SAAS,GAAG,CAAC,CAAA;IACb,UAAU,GAAG,CAAC,CAAA;IACd,OAAO,GAAG,CAAC,CAAA;IAEnB,YAAY,EACV,SAAS,EACT,KAAK,GAAG,CAAC,EACT,SAAS,GAAG,IAAI,EAChB,QAAQ,GAAG,MAAM,GACW;QAC5B,KAAK,EAAE,CAAA;QAEP,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACvD,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,QAAQ,IAAI,CAAC,CAAA;QACjD,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAA;QACzC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAE1B,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAC7B,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAA;QAE1D,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC;YAC5C,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAA;QAExE,IAAI,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM;YAC3D,MAAM,IAAI,SAAS,CACjB,iEAAiE,CAClE,CAAA;QAEH,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;YACvC,MAAM,IAAI,SAAS,CAAC,wCAAwC,CAAC,CAAA;IACjE,CAAC;IAEM,UAAU,CACf,KAAa,EACb,SAAyB,EACzB,QAA2B;QAE3B,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CACvE,CAAA;QAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACvB,CAAC;IAEM,MAAM,CAAC,QAA2B;QACvC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAE/D,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,IAAI,CAAC,MAAc;QACzB,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;YAExD,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,MAAK;YAEvB,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;YAEjC,UAAU,GAAG,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAA;YAE1C,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;YAEjB,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,KAAK,CAAC;gBAAE,SAAQ;YAEhD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,CAAA;YAEjD,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAA;QAC9D,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CACnB,UAAU,EACV,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAC5C,CAAA;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;IACxB,CAAC;IAEO,GAAG,CAAC,KAAa,EAAE,GAAW;QACpC,IAAI,IAAI,CAAC,SAAS,IAAI,GAAG,IAAI,KAAK;YAAE,OAAM;QAE1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAClC,CAAC;CACF"}
@@ -0,0 +1,8 @@
1
+ import type { Readable } from 'node:stream';
2
+ interface Options {
3
+ delimiter?: string;
4
+ encoding?: BufferEncoding;
5
+ }
6
+ export declare function csvReader(stream: Readable, options?: Options): AsyncGenerator<string, void, unknown>;
7
+ export {};
8
+ //# sourceMappingURL=csv-reader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-reader.d.ts","sourceRoot":"","sources":["../../src/helpers/csv-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAI3C,UAAU,OAAO;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B;AAED,wBAAuB,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,yCAcnE"}
@@ -0,0 +1,14 @@
1
+ import { parse } from 'csv-parse';
2
+ export async function* csvReader(stream, options) {
3
+ if (options?.encoding)
4
+ stream.setEncoding(options.encoding);
5
+ const reader = stream.pipe(parse({
6
+ delimiter: options?.delimiter,
7
+ columns: true,
8
+ relax_column_count: true,
9
+ }));
10
+ for await (const record of reader) {
11
+ yield JSON.stringify(record);
12
+ }
13
+ }
14
+ //# sourceMappingURL=csv-reader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-reader.js","sourceRoot":"","sources":["../../src/helpers/csv-reader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AAOjC,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,MAAgB,EAAE,OAAiB;IAClE,IAAI,OAAO,EAAE,QAAQ;QAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAE3D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC;QACJ,SAAS,EAAE,OAAO,EAAE,SAAS;QAC7B,OAAO,EAAE,IAAI;QACb,kBAAkB,EAAE,IAAI;KACzB,CAAC,CACH,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;AACH,CAAC"}
@@ -0,0 +1,9 @@
1
+ import type { Readable } from 'node:stream';
2
+ export type Parser = (stream: Readable, options: {
3
+ encoding: BufferEncoding;
4
+ delimiter?: string;
5
+ }) => AsyncIterable<string>;
6
+ export type ParserFormat = 'txt' | 'ndjson' | 'csv' | string;
7
+ export declare const parsers: Map<string, Parser>;
8
+ export declare function registerParser(name: ParserFormat, parser: Parser): void;
9
+ //# sourceMappingURL=parsers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/helpers/parsers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAI3C,MAAM,MAAM,MAAM,GAAG,CACnB,MAAM,EAAE,QAAQ,EAChB,OAAO,EAAE;IAAE,QAAQ,EAAE,cAAc,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KACtD,aAAa,CAAC,MAAM,CAAC,CAAA;AAE1B,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAA;AAE5D,eAAO,MAAM,OAAO,qBAQlB,CAAA;AAEF,wBAAgB,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAEvE"}
@@ -0,0 +1,14 @@
1
+ import { createInterface } from 'node:readline';
2
+ import { csvReader } from './csv-reader.js';
3
+ export const parsers = new Map([
4
+ ['txt', stream => createInterface({ input: stream, crlfDelay: Infinity })],
5
+ ['ndjson', stream => createInterface({ input: stream, crlfDelay: Infinity })],
6
+ [
7
+ 'csv',
8
+ (stream, { encoding, delimiter }) => csvReader(stream, { encoding, delimiter }),
9
+ ],
10
+ ]);
11
+ export function registerParser(name, parser) {
12
+ parsers.set(name, parser);
13
+ }
14
+ //# sourceMappingURL=parsers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/helpers/parsers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAG/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAS3C,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAuB;IACnD,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7E;QACE,KAAK;QACL,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,CAClC,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;KAC7C;CACF,CAAC,CAAA;AAEF,MAAM,UAAU,cAAc,CAAC,IAAkB,EAAE,MAAc;IAC/D,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAC3B,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { FileSystemClaim } from '../FileSystemClaim.js';
2
+ export declare function mockFileSystemClaim(implementation?: Partial<FileSystemClaim>): import("@t4h.framework/core").ClaimProvider<FileSystemClaim>;
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAmBvD,wBAAgB,mBAAmB,CACjC,cAAc,GAAE,OAAO,CAAC,eAAe,CAAM,gEAiB9C"}