@t4h.framework/fs 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +18 -3
- package/.ai/skills/framework-fs-split/SKILL.md +255 -0
- package/README.md +184 -3
- package/dist/FileSystemClaim.d.ts +4 -0
- package/dist/FileSystemClaim.d.ts.map +1 -1
- package/dist/FileSystemClaim.js.map +1 -1
- package/dist/activities/FileJoinActivity.d.ts +39 -0
- package/dist/activities/FileJoinActivity.d.ts.map +1 -0
- package/dist/activities/FileJoinActivity.js +87 -0
- package/dist/activities/FileJoinActivity.js.map +1 -0
- package/dist/activities/FileSplitActivity.d.ts +30 -28
- package/dist/activities/FileSplitActivity.d.ts.map +1 -1
- package/dist/activities/FileSplitActivity.js +39 -34
- package/dist/activities/FileSplitActivity.js.map +1 -1
- package/dist/activities/__tests__/FileJoinActivity.spec.d.ts +2 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.d.ts.map +1 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.js +160 -0
- package/dist/activities/__tests__/FileJoinActivity.spec.js.map +1 -0
- package/dist/activities/__tests__/FileSplitActivity.spec.js +114 -132
- package/dist/activities/__tests__/FileSplitActivity.spec.js.map +1 -1
- package/dist/fs.d.ts +3 -0
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +3 -0
- package/dist/fs.js.map +1 -1
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts +2 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.d.ts.map +1 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js +127 -0
- package/dist/helpers/__tests__/byte-delimiter-scanner.spec.js.map +1 -0
- package/dist/helpers/byte-delimiter-scanner.d.ts +36 -0
- package/dist/helpers/byte-delimiter-scanner.d.ts.map +1 -0
- package/dist/helpers/byte-delimiter-scanner.js +61 -0
- package/dist/helpers/byte-delimiter-scanner.js.map +1 -0
- package/package.json +8 -5
|
@@ -1,166 +1,148 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import {
|
|
2
|
+
import { buffer } from 'node:stream/consumers';
|
|
3
|
+
import { AsyncActivityResultFulfilled, AsyncActivityResultPending, Claim, WorkflowBatchRef, WorkflowClaim, } from '@t4h.framework/core';
|
|
3
4
|
import { describe, expect, it, vi } from 'vitest';
|
|
4
5
|
import { FileSystemClaim } from '../../FileSystemClaim.js';
|
|
5
|
-
import { FileSplitActivity, } from '../FileSplitActivity.js';
|
|
6
|
+
import { FileSplitActivity, Split, } from '../FileSplitActivity.js';
|
|
6
7
|
function makeBinary(content) {
|
|
7
|
-
|
|
8
|
-
const binary = {
|
|
8
|
+
return {
|
|
9
9
|
contentType: 'text/plain',
|
|
10
10
|
contentLength: Buffer.byteLength(content),
|
|
11
|
-
stream: () =>
|
|
11
|
+
stream: () => Readable.from([Buffer.from(content)]),
|
|
12
12
|
};
|
|
13
|
-
return { binary, stream };
|
|
14
13
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const
|
|
14
|
+
const childWorkflow = { id: 'child-workflow' };
|
|
15
|
+
async function bootstrapActivity(input) {
|
|
16
|
+
const durable = { id: 'durable-copy' };
|
|
17
|
+
const written = [];
|
|
18
|
+
const write = vi.fn(async (_path, readable) => {
|
|
19
|
+
written.push(await buffer(readable));
|
|
20
|
+
return durable;
|
|
21
|
+
});
|
|
22
|
+
const slice = vi.fn((binary, range, encoding) => ({ binary, range, encoding }));
|
|
23
|
+
const startMany = vi.fn(async (_workflow, inputs) => new WorkflowBatchRef('batch-1', inputs.length));
|
|
24
|
+
const result = await Claim.run([
|
|
18
25
|
{
|
|
19
26
|
provide: FileSystemClaim,
|
|
20
|
-
value: { write },
|
|
27
|
+
value: { write, slice },
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
provide: WorkflowClaim,
|
|
31
|
+
value: { startMany },
|
|
21
32
|
},
|
|
22
33
|
], () => {
|
|
23
34
|
const activity = new FileSplitActivity();
|
|
24
|
-
|
|
35
|
+
activity.id = 'activity-1';
|
|
36
|
+
return activity.bootstrap(input);
|
|
25
37
|
});
|
|
26
|
-
return {
|
|
38
|
+
return { result, durable, write, slice, startMany, written };
|
|
27
39
|
}
|
|
28
|
-
describe('
|
|
40
|
+
describe('FileSplitActivity', () => {
|
|
29
41
|
describe('toInput', () => {
|
|
30
|
-
it('
|
|
31
|
-
const activity = new FileSplitActivity();
|
|
32
|
-
const { binary } = makeBinary('');
|
|
33
|
-
const options = { format: 'csv', delimiter: ';' };
|
|
34
|
-
const input = activity.toInput(binary, options);
|
|
35
|
-
expect(input).toEqual({ binary, options });
|
|
36
|
-
});
|
|
37
|
-
it('leaves options undefined when none are provided', () => {
|
|
42
|
+
it('stores the workflow by id alongside the binary and options', () => {
|
|
38
43
|
const activity = new FileSplitActivity();
|
|
39
|
-
const
|
|
40
|
-
const
|
|
41
|
-
expect(
|
|
44
|
+
const binary = makeBinary('');
|
|
45
|
+
const options = { delimiter: Split.line() };
|
|
46
|
+
expect(activity.toInput(binary, childWorkflow, options)).toEqual({
|
|
47
|
+
binary,
|
|
48
|
+
workflow: 'child-workflow',
|
|
49
|
+
options,
|
|
50
|
+
});
|
|
42
51
|
});
|
|
43
|
-
|
|
44
|
-
describe('toOutput', () => {
|
|
45
|
-
it('returns the binaries unchanged', () => {
|
|
52
|
+
it('throws when the workflow has no id', () => {
|
|
46
53
|
const activity = new FileSplitActivity();
|
|
47
|
-
|
|
48
|
-
|
|
54
|
+
expect(() => activity.toInput(makeBinary(''), {}, {
|
|
55
|
+
delimiter: Split.line(),
|
|
56
|
+
})).toThrow(/without an id/);
|
|
49
57
|
});
|
|
50
58
|
});
|
|
51
|
-
describe('
|
|
52
|
-
it('
|
|
53
|
-
const {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
expect(write).toHaveBeenNthCalledWith(2, 'lines/1.txt', Buffer.from('bar', 'utf8'));
|
|
58
|
-
expect(write).toHaveBeenNthCalledWith(3, 'lines/2.txt', Buffer.from('baz', 'utf8'));
|
|
59
|
-
expect(binaries).toHaveLength(3);
|
|
60
|
-
});
|
|
61
|
-
it('defaults to the txt format when no options are provided', async () => {
|
|
62
|
-
const { binary } = makeBinary('only-line\n');
|
|
63
|
-
const { write } = await runActivity({ binary });
|
|
64
|
-
expect(write).toHaveBeenCalledOnce();
|
|
65
|
-
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from('only-line', 'utf8'));
|
|
66
|
-
});
|
|
67
|
-
it('treats ndjson identically to txt (line based)', async () => {
|
|
68
|
-
const { binary } = makeBinary('{"a":1}\n{"b":2}\n');
|
|
69
|
-
const { write } = await runActivity({
|
|
70
|
-
binary,
|
|
71
|
-
options: { format: 'ndjson' },
|
|
59
|
+
describe('bootstrap', () => {
|
|
60
|
+
it('persists the source once, slices by ranges, starts the batch and suspends', async () => {
|
|
61
|
+
const { result, durable, write, slice, startMany, written } = await bootstrapActivity({
|
|
62
|
+
binary: makeBinary('foo\nbar\nbaz'),
|
|
63
|
+
workflow: 'child-workflow',
|
|
64
|
+
options: { delimiter: Split.line() },
|
|
72
65
|
});
|
|
73
|
-
expect(write).
|
|
74
|
-
expect(
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
66
|
+
expect(write).toHaveBeenCalledExactlyOnceWith('split/activity-1', expect.anything());
|
|
67
|
+
expect(written[0]?.toString()).toBe('foo\nbar\nbaz');
|
|
68
|
+
expect(slice).toHaveBeenCalledTimes(3);
|
|
69
|
+
expect(slice).toHaveBeenNthCalledWith(1, durable, { start: 0, end: 3 }, 'utf8');
|
|
70
|
+
expect(slice).toHaveBeenNthCalledWith(2, durable, { start: 4, end: 7 }, 'utf8');
|
|
71
|
+
expect(slice).toHaveBeenNthCalledWith(3, durable, { start: 8, end: 11 }, 'utf8');
|
|
72
|
+
expect(startMany).toHaveBeenCalledExactlyOnceWith('child-workflow', [
|
|
73
|
+
slice.mock.results[0]?.value,
|
|
74
|
+
slice.mock.results[1]?.value,
|
|
75
|
+
slice.mock.results[2]?.value,
|
|
76
|
+
]);
|
|
77
|
+
expect(result).toBeInstanceOf(AsyncActivityResultPending);
|
|
78
|
+
expect(result.schema).toBeDefined();
|
|
79
|
+
});
|
|
80
|
+
it('splits positional records with the record-start delimiter', async () => {
|
|
81
|
+
const { slice, result } = await bootstrapActivity({
|
|
82
|
+
binary: makeBinary('10A;x\n10B;y\n10C;z'),
|
|
83
|
+
workflow: 'child-workflow',
|
|
84
|
+
options: { delimiter: Split.char('\n10', { next: 2 }) },
|
|
90
85
|
});
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
104
|
-
await runActivity({ binary });
|
|
105
|
-
expect(setEncoding).toHaveBeenCalledWith('utf8');
|
|
106
|
-
});
|
|
107
|
-
it('honors a custom encoding on both the stream and the written buffer', async () => {
|
|
108
|
-
const { binary, stream } = makeBinary('foo\n');
|
|
109
|
-
const setEncoding = vi.spyOn(stream, 'setEncoding');
|
|
110
|
-
const { write } = await runActivity({
|
|
111
|
-
binary,
|
|
112
|
-
options: { format: 'txt', encoding: 'latin1' },
|
|
86
|
+
expect(slice.mock.calls.map(([, range]) => range)).toEqual([
|
|
87
|
+
{ start: 0, end: 5 },
|
|
88
|
+
{ start: 6, end: 11 },
|
|
89
|
+
{ start: 12, end: 17 },
|
|
90
|
+
]);
|
|
91
|
+
expect(result).toBeInstanceOf(AsyncActivityResultPending);
|
|
92
|
+
});
|
|
93
|
+
it('resolves immediately for a file with no usable slices', async () => {
|
|
94
|
+
const { result, slice, startMany } = await bootstrapActivity({
|
|
95
|
+
binary: makeBinary('\n\n'),
|
|
96
|
+
workflow: 'child-workflow',
|
|
97
|
+
options: { delimiter: Split.line() },
|
|
113
98
|
});
|
|
114
|
-
expect(
|
|
115
|
-
expect(
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
options: {
|
|
99
|
+
expect(slice).not.toHaveBeenCalled();
|
|
100
|
+
expect(startMany).toHaveBeenCalledExactlyOnceWith('child-workflow', []);
|
|
101
|
+
expect(result).toEqual(new AsyncActivityResultFulfilled({ batch: 'batch-1', count: 0 }));
|
|
102
|
+
});
|
|
103
|
+
it('schedules the timeout when awaiting the batch', async () => {
|
|
104
|
+
const before = Date.now();
|
|
105
|
+
const { result } = await bootstrapActivity({
|
|
106
|
+
binary: makeBinary('foo\nbar'),
|
|
107
|
+
workflow: 'child-workflow',
|
|
108
|
+
options: { delimiter: Split.line(), timeoutInMs: 60_000 },
|
|
124
109
|
});
|
|
125
|
-
|
|
126
|
-
expect(
|
|
127
|
-
expect(
|
|
128
|
-
expect(
|
|
129
|
-
});
|
|
130
|
-
it('
|
|
131
|
-
const {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
options: {
|
|
110
|
+
const pending = result;
|
|
111
|
+
expect(pending).toBeInstanceOf(AsyncActivityResultPending);
|
|
112
|
+
expect(pending.timeoutAt?.getTime()).toBeGreaterThanOrEqual(before + 60_000);
|
|
113
|
+
expect(pending.timeoutAt?.getTime()).toBeLessThanOrEqual(Date.now() + 60_000);
|
|
114
|
+
});
|
|
115
|
+
it('suspends without a timeout when none is configured', async () => {
|
|
116
|
+
const { result } = await bootstrapActivity({
|
|
117
|
+
binary: makeBinary('foo\nbar'),
|
|
118
|
+
workflow: 'child-workflow',
|
|
119
|
+
options: { delimiter: Split.line() },
|
|
135
120
|
});
|
|
136
|
-
|
|
137
|
-
expect(write).toHaveBeenCalledOnce();
|
|
138
|
-
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from(JSON.stringify({ a: '1', b: '2' }), 'utf8'));
|
|
121
|
+
expect(result.timeoutAt).toBeUndefined();
|
|
139
122
|
});
|
|
140
|
-
it('
|
|
141
|
-
const {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
options: {
|
|
123
|
+
it('forwards the configured encoding to the scanner and the slices', async () => {
|
|
124
|
+
const { slice } = await bootstrapActivity({
|
|
125
|
+
binary: makeBinary('a;b'),
|
|
126
|
+
workflow: 'child-workflow',
|
|
127
|
+
options: { delimiter: Split.char(';'), encoding: 'latin1' },
|
|
145
128
|
});
|
|
146
|
-
expect(
|
|
147
|
-
expect(write).not.toHaveBeenCalled();
|
|
129
|
+
expect(slice).toHaveBeenCalledWith(expect.anything(), { start: 0, end: 1 }, 'latin1');
|
|
148
130
|
});
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
});
|
|
156
|
-
expect(setEncoding).toHaveBeenCalledWith('latin1');
|
|
157
|
-
expect(write).toHaveBeenCalledWith('lines/0.txt', Buffer.from(JSON.stringify({ a: '1', b: '2' }), 'latin1'));
|
|
131
|
+
});
|
|
132
|
+
describe('advance', () => {
|
|
133
|
+
it('resolves with the batch reference payload once the batch is terminal', () => {
|
|
134
|
+
const activity = new FileSplitActivity();
|
|
135
|
+
const payload = { batch: 'batch-1', count: 3 };
|
|
136
|
+
expect(activity.advance(payload)).toEqual(new AsyncActivityResultFulfilled(payload));
|
|
158
137
|
});
|
|
159
138
|
});
|
|
160
|
-
describe('
|
|
161
|
-
it('
|
|
162
|
-
const
|
|
163
|
-
|
|
139
|
+
describe('toOutput', () => {
|
|
140
|
+
it('maps the payload to a WorkflowBatchRef', () => {
|
|
141
|
+
const activity = new FileSplitActivity();
|
|
142
|
+
const result = activity.toOutput({ batch: 'batch-1', count: 7 });
|
|
143
|
+
expect(result).toBeInstanceOf(WorkflowBatchRef);
|
|
144
|
+
expect(result.id).toBe('batch-1');
|
|
145
|
+
expect(result.count).toBe(7);
|
|
164
146
|
});
|
|
165
147
|
});
|
|
166
148
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSplitActivity.spec.js","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"FileSplitActivity.spec.js","sourceRoot":"","sources":["../../../src/activities/__tests__/FileSplitActivity.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EACL,4BAA4B,EAC5B,0BAA0B,EAE1B,KAAK,EAEL,gBAAgB,EAChB,aAAa,GACd,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EACL,iBAAiB,EAEjB,KAAK,GACN,MAAM,yBAAyB,CAAA;AAEhC,SAAS,UAAU,CAAC,OAAe;IACjC,OAAO;QACL,WAAW,EAAE,YAAY;QACzB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;QACzC,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAC/B,CAAA;AACxB,CAAC;AAED,MAAM,aAAa,GAAG,EAAE,EAAE,EAAE,gBAAgB,EAAmC,CAAA;AAE/E,KAAK,UAAU,iBAAiB,CAAC,KAA6B;IAC5D,MAAM,OAAO,GAAG,EAAE,EAAE,EAAE,cAAc,EAAuB,CAAA;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,KAAa,EAAE,QAAkB,EAAE,EAAE;QAC9D,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEpC,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CACjB,CAAC,MAAc,EAAE,KAAqC,EAAE,QAAQ,EAAE,EAAE,CAClE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAsB,CACrD,CAAA;IAED,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,CACrB,KAAK,EAAE,SAAS,EAAE,MAAsB,EAAE,EAAE,CAC1C,IAAI,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CACjD,CAAA;IAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAC5B;QACE;YACE,OAAO,EAAE,eAAe;YACxB,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAgC;SACtD;QACD;YACE,OAAO,EAAE,aAAa;YACtB,KAAK,EAAE,EAAE,SAAS,EAA8B;SACjD;KACF,EACD,GAAG,EAAE;QACH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;QAExC,QAAQ,CAAC,EAAE,GAAG,YAAY,CAAA;QAE1B,OAAO,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC,CACF,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;AAC9D,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAA;YAE3C,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC/D,MAAM;gBACN,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO;aACR,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YAExC,MAAM,CAAC,GAAG,EAAE,CACV,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,EAAwB,EAAE;gBACzD,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE;aACxB,CAAC,CACH,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QAC5B,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GACzD,MAAM,iBAAiB,CAAC;gBACtB,MAAM,EAAE,UAAU,CAAC,eAAe,CAAC;gBACnC,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEJ,MAAM,CAAC,KAAK,CAAC,CAAC,+BAA+B,CAC3C,kBAAkB,EAClB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAEpD,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,MAAM,CACP,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,MAAM,CACP,CAAA;YACD,MAAM,CAAC,KAAK,CAAC,CAAC,uBAAuB,CACnC,CAAC,EACD,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,EACrB,MAAM,CACP,CAAA;YAED,MAAM,CAAC,SAAS,CAAC,CAAC,+BAA+B,CAAC,gBAAgB,EAAE;gBAClE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK;aAC7B,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;YACzD,MAAM,CAAE,MAAqC,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACrE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBAChD,MAAM,EAAE,UAAU,CAAC,qBAAqB,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE;aACxD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;gBACzD,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;gBACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;gBACrB,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACvB,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBAC3D,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC;gBAC1B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAA;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,+BAA+B,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CACpB,IAAI,4BAA4B,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACjE,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAEzB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACzC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC;gBAC9B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE;aAC1D,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,MAAoC,CAAA;YAEpD,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAA;YAC1D,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CACzD,MAAM,GAAG,MAAM,CAChB,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,mBAAmB,CACtD,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CACpB,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACzC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC;gBAC9B,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE;aACrC,CAAC,CAAA;YAEF,MAAM,CAAE,MAAqC,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAA;QAC1E,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,iBAAiB,CAAC;gBACxC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC;gBACzB,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE;aAC5D,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAChC,MAAM,CAAC,QAAQ,EAAE,EACjB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EACpB,QAAQ,CACT,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;YAC9E,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YAE9C,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACvC,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAC1C,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAA;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
package/dist/fs.d.ts
CHANGED
package/dist/fs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA"}
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA;AACpC,cAAc,kCAAkC,CAAA;AAChD,cAAc,mCAAmC,CAAA;AACjD,cAAc,qCAAqC,CAAA"}
|
package/dist/fs.js
CHANGED
package/dist/fs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA"}
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAA;AACpC,cAAc,kCAAkC,CAAA;AAChD,cAAc,mCAAmC,CAAA;AACjD,cAAc,qCAAqC,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"byte-delimiter-scanner.spec.d.ts","sourceRoot":"","sources":["../../../src/helpers/__tests__/byte-delimiter-scanner.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { buffer } from 'node:stream/consumers';
|
|
3
|
+
import { describe, expect, it } from 'vitest';
|
|
4
|
+
import { ByteDelimiterScanner, } from '../byte-delimiter-scanner.js';
|
|
5
|
+
async function scan(chunks, options) {
|
|
6
|
+
const scanner = new ByteDelimiterScanner(options);
|
|
7
|
+
const output = await buffer(Readable.from(chunks.map(chunk => Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))).pipe(scanner));
|
|
8
|
+
return { ranges: scanner.ranges, output };
|
|
9
|
+
}
|
|
10
|
+
describe('ByteDelimiterScanner', () => {
|
|
11
|
+
it('cuts at every delimiter, discarding it by default', async () => {
|
|
12
|
+
const { ranges } = await scan(['foo\nbar\nbaz'], {
|
|
13
|
+
delimiter: { value: '\n' },
|
|
14
|
+
});
|
|
15
|
+
expect(ranges).toEqual([
|
|
16
|
+
{ start: 0, end: 3 },
|
|
17
|
+
{ start: 4, end: 7 },
|
|
18
|
+
{ start: 8, end: 11 },
|
|
19
|
+
]);
|
|
20
|
+
});
|
|
21
|
+
it('skips the empty final slice when the file ends with a delimiter', async () => {
|
|
22
|
+
const { ranges } = await scan(['foo\nbar\n'], {
|
|
23
|
+
delimiter: { value: '\n' },
|
|
24
|
+
});
|
|
25
|
+
expect(ranges).toEqual([
|
|
26
|
+
{ start: 0, end: 3 },
|
|
27
|
+
{ start: 4, end: 7 },
|
|
28
|
+
]);
|
|
29
|
+
});
|
|
30
|
+
it('keeps empty slices when skipEmpty is off', async () => {
|
|
31
|
+
const { ranges } = await scan(['foo\n\nbar'], {
|
|
32
|
+
delimiter: { value: '\n' },
|
|
33
|
+
skipEmpty: false,
|
|
34
|
+
});
|
|
35
|
+
expect(ranges).toEqual([
|
|
36
|
+
{ start: 0, end: 3 },
|
|
37
|
+
{ start: 4, end: 4 },
|
|
38
|
+
{ start: 5, end: 8 },
|
|
39
|
+
]);
|
|
40
|
+
});
|
|
41
|
+
it('passes the bytes through unchanged', async () => {
|
|
42
|
+
const content = 'foo\nbar\nbaz\n';
|
|
43
|
+
const { output } = await scan([content], { delimiter: { value: '\n' } });
|
|
44
|
+
expect(output.toString()).toBe(content);
|
|
45
|
+
});
|
|
46
|
+
it('finds a delimiter split across chunk boundaries', async () => {
|
|
47
|
+
const { ranges } = await scan(['foo\r', '\nbar'], {
|
|
48
|
+
delimiter: { value: '\r\n' },
|
|
49
|
+
});
|
|
50
|
+
expect(ranges).toEqual([
|
|
51
|
+
{ start: 0, end: 3 },
|
|
52
|
+
{ start: 5, end: 8 },
|
|
53
|
+
]);
|
|
54
|
+
});
|
|
55
|
+
it('does not double count a match ending inside the kept tail', async () => {
|
|
56
|
+
// '\n' is both a full match and a possible prefix of the 2-byte tail kept
|
|
57
|
+
// between chunks; a rescan would produce a phantom empty slice.
|
|
58
|
+
const { ranges } = await scan(['a\n10b', '\n10c'], {
|
|
59
|
+
delimiter: { value: '\n10', keep: { next: 2 } },
|
|
60
|
+
});
|
|
61
|
+
expect(ranges).toEqual([
|
|
62
|
+
{ start: 0, end: 1 },
|
|
63
|
+
{ start: 2, end: 5 },
|
|
64
|
+
{ start: 6, end: 9 },
|
|
65
|
+
]);
|
|
66
|
+
});
|
|
67
|
+
it('supports the positional record-start cut: newline dies, marker stays with the next slice', async () => {
|
|
68
|
+
const content = '10A;x\n10B;y\n10C;z';
|
|
69
|
+
const { ranges } = await scan([content], {
|
|
70
|
+
delimiter: { value: '\n10', keep: { next: 2 } },
|
|
71
|
+
});
|
|
72
|
+
expect(ranges).toEqual([
|
|
73
|
+
{ start: 0, end: 5 },
|
|
74
|
+
{ start: 6, end: 11 },
|
|
75
|
+
{ start: 12, end: 17 },
|
|
76
|
+
]);
|
|
77
|
+
expect(content.slice(6, 11)).toBe('10B;y');
|
|
78
|
+
});
|
|
79
|
+
it('supports keeping delimiter bytes with the previous slice', async () => {
|
|
80
|
+
const { ranges } = await scan(['a;b;c'], {
|
|
81
|
+
delimiter: { value: ';', keep: { previous: 1 } },
|
|
82
|
+
});
|
|
83
|
+
expect(ranges).toEqual([
|
|
84
|
+
{ start: 0, end: 2 },
|
|
85
|
+
{ start: 2, end: 4 },
|
|
86
|
+
{ start: 4, end: 5 },
|
|
87
|
+
]);
|
|
88
|
+
});
|
|
89
|
+
it('cuts only every N occurrences to group records', async () => {
|
|
90
|
+
const { ranges } = await scan(['a\nb\nc\nd'], {
|
|
91
|
+
delimiter: { value: '\n' },
|
|
92
|
+
every: 2,
|
|
93
|
+
});
|
|
94
|
+
expect(ranges).toEqual([
|
|
95
|
+
{ start: 0, end: 3 },
|
|
96
|
+
{ start: 4, end: 7 },
|
|
97
|
+
]);
|
|
98
|
+
});
|
|
99
|
+
it('yields the whole file as one slice when the delimiter never occurs', async () => {
|
|
100
|
+
const { ranges } = await scan(['no-delimiter-here'], {
|
|
101
|
+
delimiter: { value: '\n' },
|
|
102
|
+
});
|
|
103
|
+
expect(ranges).toEqual([{ start: 0, end: 17 }]);
|
|
104
|
+
});
|
|
105
|
+
it('converts the delimiter to bytes using the configured encoding', async () => {
|
|
106
|
+
const { ranges } = await scan([Buffer.from('aébé', 'latin1')], {
|
|
107
|
+
delimiter: { value: 'é' },
|
|
108
|
+
encoding: 'latin1',
|
|
109
|
+
});
|
|
110
|
+
expect(ranges).toEqual([
|
|
111
|
+
{ start: 0, end: 1 },
|
|
112
|
+
{ start: 2, end: 3 },
|
|
113
|
+
]);
|
|
114
|
+
});
|
|
115
|
+
it('rejects an empty delimiter', () => {
|
|
116
|
+
expect(() => new ByteDelimiterScanner({ delimiter: { value: '' } })).toThrow(/must not be empty/);
|
|
117
|
+
});
|
|
118
|
+
it('rejects keep sizes exceeding the delimiter length', () => {
|
|
119
|
+
expect(() => new ByteDelimiterScanner({
|
|
120
|
+
delimiter: { value: ';', keep: { previous: 1, next: 1 } },
|
|
121
|
+
})).toThrow(/must not exceed/);
|
|
122
|
+
});
|
|
123
|
+
it('rejects a non-positive every', () => {
|
|
124
|
+
expect(() => new ByteDelimiterScanner({ delimiter: { value: ';' }, every: 0 })).toThrow(/positive integer/);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
//# sourceMappingURL=byte-delimiter-scanner.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"byte-delimiter-scanner.spec.js","sourceRoot":"","sources":["../../../src/helpers/__tests__/byte-delimiter-scanner.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EACL,oBAAoB,GAGrB,MAAM,8BAA8B,CAAA;AAErC,KAAK,UAAU,IAAI,CACjB,MAA2B,EAC3B,OAAoC;IAEpC,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAA;IAEjD,MAAM,MAAM,GAAG,MAAM,MAAM,CACzB,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACjB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CACpD,CACF,CAAC,IAAI,CAAC,OAAO,CAAC,CAChB,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAC3C,CAAC;AAED,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,eAAe,CAAC,EAAE;YAC/C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;SACtB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1B,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,OAAO,GAAG,iBAAiB,CAAA;QAEjC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAExE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;YAChD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;SAC7B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,0EAA0E;QAC1E,gEAAgE;QAChE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE;YACjD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAChD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0FAA0F,EAAE,KAAK,IAAI,EAAE;QACxG,MAAM,OAAO,GAAG,qBAAqB,CAAA;QAErC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE;YACvC,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAChD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;YACrB,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;SACvB,CAAC,CAAA;QAEF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE;YACvC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;SACjD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE;YAC5C,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1B,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,mBAAmB,CAAC,EAAE;YACnD,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC3B,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE;YAC7D,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE;YACzB,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YACpB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;SACrB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CACJ,GAAG,EAAE,CAAC,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAC7D,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,oBAAoB,CAAC;YACvB,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;SAC1D,CAAC,CACL,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CACJ,GAAG,EAAE,CAAC,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACxE,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Transform, type TransformCallback } from 'node:stream';
|
|
2
|
+
export type ByteRange = {
|
|
3
|
+
start: number;
|
|
4
|
+
end: number;
|
|
5
|
+
};
|
|
6
|
+
export type FileSplitDelimiter = {
|
|
7
|
+
value: string;
|
|
8
|
+
keep?: {
|
|
9
|
+
previous?: number;
|
|
10
|
+
next?: number;
|
|
11
|
+
};
|
|
12
|
+
};
|
|
13
|
+
export type ByteDelimiterScannerOptions = {
|
|
14
|
+
delimiter: FileSplitDelimiter;
|
|
15
|
+
every?: number;
|
|
16
|
+
skipEmpty?: boolean;
|
|
17
|
+
encoding?: BufferEncoding;
|
|
18
|
+
};
|
|
19
|
+
export declare class ByteDelimiterScanner extends Transform {
|
|
20
|
+
readonly ranges: ByteRange[];
|
|
21
|
+
private readonly delimiter;
|
|
22
|
+
private readonly keepPrevious;
|
|
23
|
+
private readonly keepNext;
|
|
24
|
+
private readonly cutEvery;
|
|
25
|
+
private readonly skipEmpty;
|
|
26
|
+
private pending;
|
|
27
|
+
private pendingAt;
|
|
28
|
+
private sliceStart;
|
|
29
|
+
private matches;
|
|
30
|
+
constructor({ delimiter, every, skipEmpty, encoding, }: ByteDelimiterScannerOptions);
|
|
31
|
+
_transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback): void;
|
|
32
|
+
_flush(callback: TransformCallback): void;
|
|
33
|
+
private scan;
|
|
34
|
+
private cut;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=byte-delimiter-scanner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"byte-delimiter-scanner.d.ts","sourceRoot":"","sources":["../../src/helpers/byte-delimiter-scanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,IAAI,CAAC,EAAE,MAAM,CAAA;KACd,CAAA;CACF,CAAA;AAED,MAAM,MAAM,2BAA2B,GAAG;IACxC,SAAS,EAAE,kBAAkB,CAAA;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B,CAAA;AAED,qBAAa,oBAAqB,SAAQ,SAAS;IACjD,SAAgB,MAAM,EAAE,SAAS,EAAE,CAAK;IAExC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,OAAO,CAAI;gBAEP,EACV,SAAS,EACT,KAAS,EACT,SAAgB,EAChB,QAAiB,GAClB,EAAE,2BAA2B;IAwBvB,UAAU,CACf,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,iBAAiB;IAStB,MAAM,CAAC,QAAQ,EAAE,iBAAiB;IAMzC,OAAO,CAAC,IAAI;IA8BZ,OAAO,CAAC,GAAG;CAKZ"}
|