@t4h.framework/fs 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/.ai/skills/framework-fs/SKILL.md +28 -7
  2. package/.ai/skills/framework-fs-split/SKILL.md +113 -18
  3. package/.ai/skills/framework-fs-write/SKILL.md +242 -0
  4. package/CHANGELOG.md +78 -0
  5. package/LICENSE +21 -0
  6. package/README.md +154 -7
  7. package/dist/FileSystemClaim.d.ts +1 -1
  8. package/dist/FileSystemClaim.d.ts.map +1 -1
  9. package/dist/FileSystemClaim.js.map +1 -1
  10. package/dist/activities/FileJoinActivity.d.ts +32 -12
  11. package/dist/activities/FileJoinActivity.d.ts.map +1 -1
  12. package/dist/activities/FileJoinActivity.js +21 -34
  13. package/dist/activities/FileJoinActivity.js.map +1 -1
  14. package/dist/activities/FileReadActivity.d.ts +22 -0
  15. package/dist/activities/FileReadActivity.d.ts.map +1 -0
  16. package/dist/activities/FileReadActivity.js +68 -0
  17. package/dist/activities/FileReadActivity.js.map +1 -0
  18. package/dist/activities/FileSplitActivity.d.ts +2 -1
  19. package/dist/activities/FileSplitActivity.d.ts.map +1 -1
  20. package/dist/activities/FileSplitActivity.js +1 -0
  21. package/dist/activities/FileSplitActivity.js.map +1 -1
  22. package/dist/activities/FileWriteActivity.d.ts +20 -0
  23. package/dist/activities/FileWriteActivity.d.ts.map +1 -0
  24. package/dist/activities/FileWriteActivity.js +26 -0
  25. package/dist/activities/FileWriteActivity.js.map +1 -0
  26. package/dist/constants/operations.d.ts +3 -0
  27. package/dist/constants/operations.d.ts.map +1 -0
  28. package/dist/constants/operations.js +18 -0
  29. package/dist/constants/operations.js.map +1 -0
  30. package/dist/fs.d.ts +2 -0
  31. package/dist/fs.d.ts.map +1 -1
  32. package/dist/fs.js +2 -0
  33. package/dist/fs.js.map +1 -1
  34. package/dist/helpers/CSV.d.ts +14 -0
  35. package/dist/helpers/CSV.d.ts.map +1 -0
  36. package/dist/helpers/CSV.js +44 -0
  37. package/dist/helpers/CSV.js.map +1 -0
  38. package/dist/helpers/byte-delimiter-scanner.d.ts +8 -1
  39. package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -1
  40. package/dist/helpers/byte-delimiter-scanner.js +26 -2
  41. package/dist/helpers/byte-delimiter-scanner.js.map +1 -1
  42. package/dist/tools/assert-where-serializable.d.ts +2 -0
  43. package/dist/tools/assert-where-serializable.d.ts.map +1 -0
  44. package/dist/tools/assert-where-serializable.js +19 -0
  45. package/dist/tools/assert-where-serializable.js.map +1 -0
  46. package/package.json +6 -3
  47. package/dist/ActivityFileSystemClaim.d.ts +0 -7
  48. package/dist/ActivityFileSystemClaim.d.ts.map +0 -1
  49. package/dist/ActivityFileSystemClaim.js +0 -3
  50. package/dist/ActivityFileSystemClaim.js.map +0 -1
  51. package/dist/activities/__tests__/FileJoinActivity.spec.d.ts +0 -2
  52. package/dist/activities/__tests__/FileJoinActivity.spec.d.ts.map +0 -1
  53. package/dist/activities/__tests__/FileJoinActivity.spec.js +0 -160
  54. package/dist/activities/__tests__/FileJoinActivity.spec.js.map +0 -1
  55. package/dist/activities/__tests__/FileSplitActivity.spec.d.ts +0 -2
  56. package/dist/activities/__tests__/FileSplitActivity.spec.d.ts.map +0 -1
  57. package/dist/activities/__tests__/FileSplitActivity.spec.js +0 -149
  58. package/dist/activities/__tests__/FileSplitActivity.spec.js.map +0 -1
  59. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts +0 -2
  60. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts.map +0 -1
  61. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +0 -127
  62. package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +0 -1
  63. package/dist/helpers/__tests__/csv-reader.spec.d.ts +0 -2
  64. package/dist/helpers/__tests__/csv-reader.spec.d.ts.map +0 -1
  65. package/dist/helpers/__tests__/csv-reader.spec.js +0 -60
  66. package/dist/helpers/__tests__/csv-reader.spec.js.map +0 -1
  67. package/dist/helpers/csv-reader.d.ts +0 -8
  68. package/dist/helpers/csv-reader.d.ts.map +0 -1
  69. package/dist/helpers/csv-reader.js +0 -14
  70. package/dist/helpers/csv-reader.js.map +0 -1
  71. package/dist/helpers/parsers.d.ts +0 -9
  72. package/dist/helpers/parsers.d.ts.map +0 -1
  73. package/dist/helpers/parsers.js +0 -14
  74. package/dist/helpers/parsers.js.map +0 -1
  75. package/dist/testing/index.d.ts +0 -3
  76. package/dist/testing/index.d.ts.map +0 -1
  77. package/dist/testing/index.js +0 -32
  78. package/dist/testing/index.js.map +0 -1
@@ -1,149 +0,0 @@
1
- import { Readable } from 'node:stream';
2
- import { buffer } from 'node:stream/consumers';
3
- import { AsyncActivityResultFulfilled, AsyncActivityResultPending, Claim, WorkflowBatchRef, WorkflowClaim, } from '@t4h.framework/core';
4
- import { describe, expect, it, vi } from 'vitest';
5
- import { FileSystemClaim } from '../../FileSystemClaim.js';
6
- import { FileSplitActivity, Split, } from '../FileSplitActivity.js';
7
- function makeBinary(content) {
8
- return {
9
- contentType: 'text/plain',
10
- contentLength: Buffer.byteLength(content),
11
- stream: () => Readable.from([Buffer.from(content)]),
12
- };
13
- }
14
- const childWorkflow = { id: 'child-workflow' };
15
- async function bootstrapActivity(input) {
16
- const durable = { id: 'durable-copy' };
17
- const written = [];
18
- const write = vi.fn(async (_path, readable) => {
19
- written.push(await buffer(readable));
20
- return durable;
21
- });
22
- const slice = vi.fn((binary, range, encoding) => ({ binary, range, encoding }));
23
- const startMany = vi.fn(async (_workflow, inputs) => new WorkflowBatchRef('batch-1', inputs.length));
24
- const result = await Claim.run([
25
- {
26
- provide: FileSystemClaim,
27
- value: { write, slice },
28
- },
29
- {
30
- provide: WorkflowClaim,
31
- value: { startMany },
32
- },
33
- ], () => {
34
- const activity = new FileSplitActivity();
35
- activity.id = 'activity-1';
36
- return activity.bootstrap(input);
37
- });
38
- return { result, durable, write, slice, startMany, written };
39
- }
40
- describe('FileSplitActivity', () => {
41
- describe('toInput', () => {
42
- it('stores the workflow by id alongside the binary and options', () => {
43
- const activity = new FileSplitActivity();
44
- const binary = makeBinary('');
45
- const options = { delimiter: Split.line() };
46
- expect(activity.toInput(binary, childWorkflow, options)).toEqual({
47
- binary,
48
- workflow: 'child-workflow',
49
- options,
50
- });
51
- });
52
- it('throws when the workflow has no id', () => {
53
- const activity = new FileSplitActivity();
54
- expect(() => activity.toInput(makeBinary(''), {}, {
55
- delimiter: Split.line(),
56
- })).toThrow(/without an id/);
57
- });
58
- });
59
- describe('bootstrap', () => {
60
- it('persists the source once, slices by ranges, starts the batch and suspends', async () => {
61
- const { result, durable, write, slice, startMany, written } = await bootstrapActivity({
62
- binary: makeBinary('foo\nbar\nbaz'),
63
- workflow: 'child-workflow',
64
- options: { delimiter: Split.line() },
65
- });
66
- expect(write).toHaveBeenCalledExactlyOnceWith('split/activity-1', expect.anything());
67
- expect(written[0]?.toString()).toBe('foo\nbar\nbaz');
68
- expect(slice).toHaveBeenCalledTimes(3);
69
- expect(slice).toHaveBeenNthCalledWith(1, durable, { start: 0, end: 3 }, 'utf8');
70
- expect(slice).toHaveBeenNthCalledWith(2, durable, { start: 4, end: 7 }, 'utf8');
71
- expect(slice).toHaveBeenNthCalledWith(3, durable, { start: 8, end: 11 }, 'utf8');
72
- expect(startMany).toHaveBeenCalledExactlyOnceWith('child-workflow', [
73
- slice.mock.results[0]?.value,
74
- slice.mock.results[1]?.value,
75
- slice.mock.results[2]?.value,
76
- ]);
77
- expect(result).toBeInstanceOf(AsyncActivityResultPending);
78
- expect(result.schema).toBeDefined();
79
- });
80
- it('splits positional records with the record-start delimiter', async () => {
81
- const { slice, result } = await bootstrapActivity({
82
- binary: makeBinary('10A;x\n10B;y\n10C;z'),
83
- workflow: 'child-workflow',
84
- options: { delimiter: Split.char('\n10', { next: 2 }) },
85
- });
86
- expect(slice.mock.calls.map(([, range]) => range)).toEqual([
87
- { start: 0, end: 5 },
88
- { start: 6, end: 11 },
89
- { start: 12, end: 17 },
90
- ]);
91
- expect(result).toBeInstanceOf(AsyncActivityResultPending);
92
- });
93
- it('resolves immediately for a file with no usable slices', async () => {
94
- const { result, slice, startMany } = await bootstrapActivity({
95
- binary: makeBinary('\n\n'),
96
- workflow: 'child-workflow',
97
- options: { delimiter: Split.line() },
98
- });
99
- expect(slice).not.toHaveBeenCalled();
100
- expect(startMany).toHaveBeenCalledExactlyOnceWith('child-workflow', []);
101
- expect(result).toEqual(new AsyncActivityResultFulfilled({ batch: 'batch-1', count: 0 }));
102
- });
103
- it('schedules the timeout when awaiting the batch', async () => {
104
- const before = Date.now();
105
- const { result } = await bootstrapActivity({
106
- binary: makeBinary('foo\nbar'),
107
- workflow: 'child-workflow',
108
- options: { delimiter: Split.line(), timeoutInMs: 60_000 },
109
- });
110
- const pending = result;
111
- expect(pending).toBeInstanceOf(AsyncActivityResultPending);
112
- expect(pending.timeoutAt?.getTime()).toBeGreaterThanOrEqual(before + 60_000);
113
- expect(pending.timeoutAt?.getTime()).toBeLessThanOrEqual(Date.now() + 60_000);
114
- });
115
- it('suspends without a timeout when none is configured', async () => {
116
- const { result } = await bootstrapActivity({
117
- binary: makeBinary('foo\nbar'),
118
- workflow: 'child-workflow',
119
- options: { delimiter: Split.line() },
120
- });
121
- expect(result.timeoutAt).toBeUndefined();
122
- });
123
- it('forwards the configured encoding to the scanner and the slices', async () => {
124
- const { slice } = await bootstrapActivity({
125
- binary: makeBinary('a;b'),
126
- workflow: 'child-workflow',
127
- options: { delimiter: Split.char(';'), encoding: 'latin1' },
128
- });
129
- expect(slice).toHaveBeenCalledWith(expect.anything(), { start: 0, end: 1 }, 'latin1');
130
- });
131
- });
132
- describe('advance', () => {
133
- it('resolves with the batch reference payload once the batch is terminal', () => {
134
- const activity = new FileSplitActivity();
135
- const payload = { batch: 'batch-1', count: 3 };
136
- expect(activity.advance(payload)).toEqual(new AsyncActivityResultFulfilled(payload));
137
- });
138
- });
139
- describe('toOutput', () => {
140
- it('maps the payload to a WorkflowBatchRef', () => {
141
- const activity = new FileSplitActivity();
142
- const result = activity.toOutput({ batch: 'batch-1', count: 7 });
143
- expect(result).toBeInstanceOf(WorkflowBatchRef);
144
- expect(result.id).toBe('batch-1');
145
- expect(result.count).toBe(7);
146
- });
147
- });
148
- });
149
- //# sourceMappingURL=FileSplitActivity.spec.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"FileSplitActivity.spec.js","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EACL,4BAA4B,EAC5B,0BAA0B,EAE1B,KAAK,EAEL,gBAAgB,EAChB,aAAa,GACd,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EACL,iBAAiB,EAEjB,KAAK,GACN,MAAM,yBAAyB,CAAA;AAEhC,SAAS,UAAU,CAAC,OAAe;IACjC,OAAO;QACL,WAAW,EAAE,YAAY;QACzB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;QACzC,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAC/B,CAAA;AACxB,CAAC;AAED,MAAM,aAAa,GAAG,EAAE,EAAE,EAAE,gBAAgB,EAAmC,CAAA;AAE/E,KAAK,UAAU,iBAAiB,CAAC,KAA6B;IAC5D,MAAM,OAAO,GAAG,EAAE,EAAE,EAAE,cAAc,EAAuB,CAAA;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,KAAa,EAAE,QAAkB,EAAE,EAAE;QAC9D,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEpC,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CACjB,CAAC,MAAc,EAAE,KAAqC,EAAE,QAAQ,EAAE,EAAE,CAClE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAsB,CACrD,CAAA;IAED,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,CACrB,KAAK,EAAE,SAAS,EAAE,MAAsB,EAAE,EAAE,CAC1C,IAAI,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CACjD,CAAA;IAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAC5B;QACE;YACE,OAAO,EAAE,eAAe;YACxB,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAgC;SACtD;QACD;YACE,OAAO,EAAE,aAAa;YACtB,KAAK,EAAE,EAAE,SAAS,EAA8B;SACjD;KACF,EACD,GAAG,EAAE;QACH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;QAExC,QAAQ,CAAC,EAAE,GAAG,YAAY,CAAA;QAE1B,OAAO,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC,CACF,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;AAC9D,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAA;YAE3C,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC/D,MAAM;gBACN,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO;aACR,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YAExC,MAAM,CAAC,GAAG,EAAE,CACV,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,EAAwB,EAAE;gBACzD,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE;aACxB,CAAC,CACH,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QAC5B,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GACzD,MAAM,iBAAiB,CAAC;gBACtB,MAAM,EAAE,UAAU,CAAC,eAAe,CAAC;gBACnC,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEJ,MAAM,CAAC,KAAK,CAAC,CAAC,+BAA+B,CAC3C,kBAAkB,EAClB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAEpD,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,MAAM,CACP,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,MAAM,CACP,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,EACrB,MAAM,CACP,CAAA;YAED,MAAM,CAAC,SAAS,CAAC,CAAC,+BAA+B,CAAC,gBAAgB,EAAE;gBAClE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;aAC7B,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;YACzD,MAAM,CAAE,MAAqC,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACrE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBAChD,MAAM,EAAE,UAAU,CAAC,qBAAqB,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE;aACxD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;gBACzD,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;gBACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;gBACrB,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACvB,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBAC3D,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC;gBAC1B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,+BAA+B,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CACpB,IAAI,4BAA4B,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACjE,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAEzB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACzC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC;gBAC9B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE;aAC1D,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,MAAoC,CAAA;YAEpD,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;YAC1D,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CACzD,MAAM,GAAG,MAAM,CAChB,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,mBAAmB,CACtD,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CACpB,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACzC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC;gBAC9B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEF,MAAM,CAAE,MAAqC,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAA;QAC1E,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACxC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC;gBACzB,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE;aAC5D,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,MAAM,CAAC,QAAQ,EAAE,EACjB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,QAAQ,CACT,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;YAC9E,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YAE9C,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACvC,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAC1C,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=byte-delimiter-scanner.spec.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"byte-delimiter-scanner.spec.d.ts","sourceRoot":"","sources":["../../../src/helpers/__tests__/byte-delimiter-scanner.spec.ts"],"names":[],"mappings":""}
@@ -1,127 +0,0 @@
1
- import { Readable } from 'node:stream';
2
- import { buffer } from 'node:stream/consumers';
3
- import { describe, expect, it } from 'vitest';
4
- import { ByteDelimiterScanner, } from '../byte-delimiter-scanner.js';
5
- async function scan(chunks, options) {
6
- const scanner = new ByteDelimiterScanner(options);
7
- const output = await buffer(Readable.from(chunks.map(chunk => Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))).pipe(scanner));
8
- return { ranges: scanner.ranges, output };
9
- }
10
- describe('ByteDelimiterScanner', () => {
11
- it('cuts at every delimiter, discarding it by default', async () => {
12
- const { ranges } = await scan(['foo\nbar\nbaz'], {
13
- delimiter: { value: '\n' },
14
- });
15
- expect(ranges).toEqual([
16
- { start: 0, end: 3 },
17
- { start: 4, end: 7 },
18
- { start: 8, end: 11 },
19
- ]);
20
- });
21
- it('skips the empty final slice when the file ends with a delimiter', async () => {
22
- const { ranges } = await scan(['foo\nbar\n'], {
23
- delimiter: { value: '\n' },
24
- });
25
- expect(ranges).toEqual([
26
- { start: 0, end: 3 },
27
- { start: 4, end: 7 },
28
- ]);
29
- });
30
- it('keeps empty slices when skipEmpty is off', async () => {
31
- const { ranges } = await scan(['foo\n\nbar'], {
32
- delimiter: { value: '\n' },
33
- skipEmpty: false,
34
- });
35
- expect(ranges).toEqual([
36
- { start: 0, end: 3 },
37
- { start: 4, end: 4 },
38
- { start: 5, end: 8 },
39
- ]);
40
- });
41
- it('passes the bytes through unchanged', async () => {
42
- const content = 'foo\nbar\nbaz\n';
43
- const { output } = await scan([content], { delimiter: { value: '\n' } });
44
- expect(output.toString()).toBe(content);
45
- });
46
- it('finds a delimiter split across chunk boundaries', async () => {
47
- const { ranges } = await scan(['foo\r', '\nbar'], {
48
- delimiter: { value: '\r\n' },
49
- });
50
- expect(ranges).toEqual([
51
- { start: 0, end: 3 },
52
- { start: 5, end: 8 },
53
- ]);
54
- });
55
- it('does not double count a match ending inside the kept tail', async () => {
56
- // '\n' is both a full match and a possible prefix of the 2-byte tail kept
57
- // between chunks; a rescan would produce a phantom empty slice.
58
- const { ranges } = await scan(['a\n10b', '\n10c'], {
59
- delimiter: { value: '\n10', keep: { next: 2 } },
60
- });
61
- expect(ranges).toEqual([
62
- { start: 0, end: 1 },
63
- { start: 2, end: 5 },
64
- { start: 6, end: 9 },
65
- ]);
66
- });
67
- it('supports the positional record-start cut: newline dies, marker stays with the next slice', async () => {
68
- const content = '10A;x\n10B;y\n10C;z';
69
- const { ranges } = await scan([content], {
70
- delimiter: { value: '\n10', keep: { next: 2 } },
71
- });
72
- expect(ranges).toEqual([
73
- { start: 0, end: 5 },
74
- { start: 6, end: 11 },
75
- { start: 12, end: 17 },
76
- ]);
77
- expect(content.slice(6, 11)).toBe('10B;y');
78
- });
79
- it('supports keeping delimiter bytes with the previous slice', async () => {
80
- const { ranges } = await scan(['a;b;c'], {
81
- delimiter: { value: ';', keep: { previous: 1 } },
82
- });
83
- expect(ranges).toEqual([
84
- { start: 0, end: 2 },
85
- { start: 2, end: 4 },
86
- { start: 4, end: 5 },
87
- ]);
88
- });
89
- it('cuts only every N occurrences to group records', async () => {
90
- const { ranges } = await scan(['a\nb\nc\nd'], {
91
- delimiter: { value: '\n' },
92
- every: 2,
93
- });
94
- expect(ranges).toEqual([
95
- { start: 0, end: 3 },
96
- { start: 4, end: 7 },
97
- ]);
98
- });
99
- it('yields the whole file as one slice when the delimiter never occurs', async () => {
100
- const { ranges } = await scan(['no-delimiter-here'], {
101
- delimiter: { value: '\n' },
102
- });
103
- expect(ranges).toEqual([{ start: 0, end: 17 }]);
104
- });
105
- it('converts the delimiter to bytes using the configured encoding', async () => {
106
- const { ranges } = await scan([Buffer.from('aébé', 'latin1')], {
107
- delimiter: { value: 'é' },
108
- encoding: 'latin1',
109
- });
110
- expect(ranges).toEqual([
111
- { start: 0, end: 1 },
112
- { start: 2, end: 3 },
113
- ]);
114
- });
115
- it('rejects an empty delimiter', () => {
116
- expect(() => new ByteDelimiterScanner({ delimiter: { value: '' } })).toThrow(/must not be empty/);
117
- });
118
- it('rejects keep sizes exceeding the delimiter length', () => {
119
- expect(() => new ByteDelimiterScanner({
120
- delimiter: { value: ';', keep: { previous: 1, next: 1 } },
121
- })).toThrow(/must not exceed/);
122
- });
123
- it('rejects a non-positive every', () => {
124
- expect(() => new ByteDelimiterScanner({ delimiter: { value: ';' }, every: 0 })).toThrow(/positive integer/);
125
- });
126
- });
127
- //# sourceMappingURL=byte-delimiter-scanner.spec.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"byte-delimiter-scanner.spec.js","sourceRoot":"","sources":["../../../src/helpers/__tests__/byte-delimiter-scanner.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EACL,oBAAoB,GAGrB,MAAM,8BAA8B,CAAA;AAErC,KAAK,UAAU,IAAI,CACjB,MAA2B,EAC3B,OAAoC;IAEpC,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAA;IAEjD,MAAM,MAAM,GAAG,MAAM,MAAM,CACzB,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACjB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CACpD,CACF,CAAC,IAAI,CAAC,OAAO,CAAC,CAChB,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAC3C,CAAC;AAED,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,eAAe,CAAC,EAAE;YAC/C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;SACtB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1B,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,OAAO,GAAG,iBAAiB,CAAA;QAEjC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAExE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;YAChD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;SAC7B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,0EAA0E;QAC1E,gEAAgE;QAChE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE;YACjD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAChD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0FAA0F,EAAE,KAAK,IAAI,EAAE;QACxG,MAAM,OAAO,GAAG,qBAAqB,CAAA;QAErC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE;YACvC,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAChD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;YACrB,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;SACvB,CAAC,CAAA;QAEF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE;YACvC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;SACjD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1B,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,mBAAmB,CAAC,EAAE;YACnD,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE;YAC7D,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE;YACzB,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CACJ,GAAG,EAAE,CAAC,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAC7D,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,oBAAoB,CAAC;YACvB,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAC1D,CAAC,CACL,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CACJ,GAAG,EAAE,CAAC,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACxE,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=csv-reader.spec.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"csv-reader.spec.d.ts","sourceRoot":"","sources":["../../../src/helpers/__tests__/csv-reader.spec.ts"],"names":[],"mappings":""}
@@ -1,60 +0,0 @@
1
- import { Readable } from 'node:stream';
2
- import { describe, expect, it, vi } from 'vitest';
3
- import { csvReader } from '../csv-reader.js';
4
- async function collect(iterable) {
5
- const records = [];
6
- for await (const record of iterable)
7
- records.push(record);
8
- return records;
9
- }
10
- describe('csvReader', () => {
11
- it('parses rows into objects keyed by the header row', async () => {
12
- const stream = Readable.from(['name,age\nalice,30\nbob,25\n']);
13
- const records = await collect(csvReader(stream));
14
- expect(records).toEqual([
15
- JSON.stringify({ name: 'alice', age: '30' }),
16
- JSON.stringify({ name: 'bob', age: '25' }),
17
- ]);
18
- });
19
- it('uses comma as the default delimiter', async () => {
20
- const stream = Readable.from(['a,b\n1,2\n']);
21
- const records = await collect(csvReader(stream));
22
- expect(records).toEqual([JSON.stringify({ a: '1', b: '2' })]);
23
- });
24
- it('respects a custom delimiter', async () => {
25
- const stream = Readable.from(['a;b\n1;2\n']);
26
- const records = await collect(csvReader(stream, { delimiter: ';' }));
27
- expect(records).toEqual([JSON.stringify({ a: '1', b: '2' })]);
28
- });
29
- it('tolerates rows with a mismatched column count (relax_column_count)', async () => {
30
- const stream = Readable.from(['a,b\n1\n2,3,4\n']);
31
- const records = await collect(csvReader(stream, { delimiter: ',' }));
32
- expect(records).toEqual([
33
- JSON.stringify({ a: '1' }),
34
- JSON.stringify({ a: '2', b: '3' }),
35
- ]);
36
- });
37
- it('yields nothing for an empty stream', async () => {
38
- const stream = Readable.from(['']);
39
- const records = await collect(csvReader(stream));
40
- expect(records).toEqual([]);
41
- });
42
- it('yields nothing when only the header row is present', async () => {
43
- const stream = Readable.from(['a,b\n']);
44
- const records = await collect(csvReader(stream));
45
- expect(records).toEqual([]);
46
- });
47
- it('sets the stream encoding when the encoding option is provided', async () => {
48
- const stream = Readable.from(['a,b\n1,2\n']);
49
- const setEncoding = vi.spyOn(stream, 'setEncoding');
50
- await collect(csvReader(stream, { encoding: 'latin1' }));
51
- expect(setEncoding).toHaveBeenCalledWith('latin1');
52
- });
53
- it('does not touch the stream encoding when no encoding option is provided', async () => {
54
- const stream = Readable.from(['a,b\n1,2\n']);
55
- const setEncoding = vi.spyOn(stream, 'setEncoding');
56
- await collect(csvReader(stream));
57
- expect(setEncoding).not.toHaveBeenCalled();
58
- });
59
- });
60
- //# sourceMappingURL=csv-reader.spec.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"csv-reader.spec.js","sourceRoot":"","sources":["../../../src/helpers/__tests__/csv-reader.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAE5C,KAAK,UAAU,OAAO,CAAI,QAA0B;IAClD,MAAM,OAAO,GAAQ,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEzD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAA;QAE9D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;SAC3C,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAA;QAEjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;SACnC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAElC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;QAEvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QAEnD,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAA;QAExD,MAAM,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QAEnD,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAEhC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;IAC5C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -1,8 +0,0 @@
1
- import type { Readable } from 'node:stream';
2
- interface Options {
3
- delimiter?: string;
4
- encoding?: BufferEncoding;
5
- }
6
- export declare function csvReader(stream: Readable, options?: Options): AsyncGenerator<string, void, unknown>;
7
- export {};
8
- //# sourceMappingURL=csv-reader.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"csv-reader.d.ts","sourceRoot":"","sources":["../../src/helpers/csv-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAI3C,UAAU,OAAO;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B;AAED,wBAAuB,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,yCAcnE"}
@@ -1,14 +0,0 @@
1
- import { parse } from 'csv-parse';
2
- export async function* csvReader(stream, options) {
3
- if (options?.encoding)
4
- stream.setEncoding(options.encoding);
5
- const reader = stream.pipe(parse({
6
- delimiter: options?.delimiter,
7
- columns: true,
8
- relax_column_count: true,
9
- }));
10
- for await (const record of reader) {
11
- yield JSON.stringify(record);
12
- }
13
- }
14
- //# sourceMappingURL=csv-reader.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"csv-reader.js","sourceRoot":"","sources":["../../src/helpers/csv-reader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AAOjC,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,MAAgB,EAAE,OAAiB;IAClE,IAAI,OAAO,EAAE,QAAQ;QAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAE3D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC;QACJ,SAAS,EAAE,OAAO,EAAE,SAAS;QAC7B,OAAO,EAAE,IAAI;QACb,kBAAkB,EAAE,IAAI;KACzB,CAAC,CACH,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;AACH,CAAC"}
@@ -1,9 +0,0 @@
1
- import type { Readable } from 'node:stream';
2
- export type Parser = (stream: Readable, options: {
3
- encoding: BufferEncoding;
4
- delimiter?: string;
5
- }) => AsyncIterable<string>;
6
- export type ParserFormat = 'txt' | 'ndjson' | 'csv' | string;
7
- export declare const parsers: Map<string, Parser>;
8
- export declare function registerParser(name: ParserFormat, parser: Parser): void;
9
- //# sourceMappingURL=parsers.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/helpers/parsers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAI3C,MAAM,MAAM,MAAM,GAAG,CACnB,MAAM,EAAE,QAAQ,EAChB,OAAO,EAAE;IAAE,QAAQ,EAAE,cAAc,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KACtD,aAAa,CAAC,MAAM,CAAC,CAAA;AAE1B,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAA;AAE5D,eAAO,MAAM,OAAO,qBAQlB,CAAA;AAEF,wBAAgB,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAEvE"}
@@ -1,14 +0,0 @@
1
- import { createInterface } from 'node:readline';
2
- import { csvReader } from './csv-reader.js';
3
- export const parsers = new Map([
4
- ['txt', stream => createInterface({ input: stream, crlfDelay: Infinity })],
5
- ['ndjson', stream => createInterface({ input: stream, crlfDelay: Infinity })],
6
- [
7
- 'csv',
8
- (stream, { encoding, delimiter }) => csvReader(stream, { encoding, delimiter }),
9
- ],
10
- ]);
11
- export function registerParser(name, parser) {
12
- parsers.set(name, parser);
13
- }
14
- //# sourceMappingURL=parsers.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/helpers/parsers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAG/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAS3C,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAuB;IACnD,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7E;QACE,KAAK;QACL,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,CAClC,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;KAC7C;CACF,CAAC,CAAA;AAEF,MAAM,UAAU,cAAc,CAAC,IAAkB,EAAE,MAAc;IAC/D,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAC3B,CAAC"}
@@ -1,3 +0,0 @@
1
- import { FileSystemClaim } from '../FileSystemClaim.js';
2
- export declare function mockFileSystemClaim(implementation?: Partial<FileSystemClaim>): import("@t4h.framework/core").ClaimProvider<FileSystemClaim>;
3
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAmBvD,wBAAgB,mBAAmB,CACjC,cAAc,GAAE,OAAO,CAAC,eAAe,CAAM,gEAiB9C"}
@@ -1,32 +0,0 @@
1
- import { Readable } from 'node:stream';
2
- import { Binary } from '@t4h.framework/core';
3
- import { mockClaim } from '@t4h.framework/core/testing';
4
- import { FileSystemClaim } from '../FileSystemClaim.js';
5
- class MemoryBinary extends Binary {
6
- data;
7
- contentType;
8
- constructor(data, contentType) {
9
- super();
10
- this.data = data;
11
- this.contentType = contentType;
12
- }
13
- get contentLength() {
14
- return this.data.byteLength;
15
- }
16
- stream() {
17
- return Readable.from(this.data);
18
- }
19
- }
20
- export function mockFileSystemClaim(implementation = {}) {
21
- return mockClaim(FileSystemClaim, {
22
- write: async (_filepath, readable) => {
23
- const chunks = [];
24
- for await (const chunk of readable)
25
- chunks.push(Buffer.from(chunk));
26
- return new MemoryBinary(Buffer.concat(chunks), 'application/octet-stream');
27
- },
28
- read: async (filepath) => new MemoryBinary(Buffer.from(filepath), 'application/octet-stream'),
29
- ...implementation,
30
- });
31
- }
32
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AAEvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAEvD,MAAM,YAAa,SAAQ,MAAM;IAEZ;IACD;IAFlB,YACmB,IAAY,EACb,WAAmB;QAEnC,KAAK,EAAE,CAAA;QAHU,SAAI,GAAJ,IAAI,CAAQ;QACb,gBAAW,GAAX,WAAW,CAAQ;IAGrC,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAA;IAC7B,CAAC;IAEM,MAAM;QACX,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;CACF;AAED,MAAM,UAAU,mBAAmB,CACjC,iBAA2C,EAAE;IAE7C,OAAO,SAAS,CAAC,eAAe,EAAE;QAChC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;YACnC,MAAM,MAAM,GAAa,EAAE,CAAA;YAE3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ;gBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;YAEnE,OAAO,IAAI,YAAY,CACrB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EACrB,0BAA0B,CAC3B,CAAA;QACH,CAAC;QACD,IAAI,EAAE,KAAK,EAAC,QAAQ,EAAC,EAAE,CACrB,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;QACrE,GAAG,cAAc;KAClB,CAAC,CAAA;AACJ,CAAC"}