@t4h.framework/fs 0.12.2 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/framework-fs/SKILL.md +25 -26
- package/.ai/skills/framework-fs-split/SKILL.md +40 -54
- package/.ai/skills/framework-fs-write/SKILL.md +13 -15
- package/CHANGELOG.md +44 -0
- package/README.md +14 -41
- package/dist/activities/FileJoinActivity.d.ts +3 -3
- package/dist/activities/FileJoinActivity.d.ts.map +1 -1
- package/dist/activities/FileJoinActivity.js.map +1 -1
- package/dist/activities/FileReadActivity.d.ts +1 -0
- package/dist/activities/FileReadActivity.d.ts.map +1 -1
- package/dist/activities/FileReadActivity.js +24 -19
- package/dist/activities/FileReadActivity.js.map +1 -1
- package/dist/activities/FileSplitActivity.d.ts +8 -15
- package/dist/activities/FileSplitActivity.d.ts.map +1 -1
- package/dist/activities/FileSplitActivity.js +18 -32
- package/dist/activities/FileSplitActivity.js.map +1 -1
- package/dist/constants/operations.d.ts +1 -1
- package/dist/constants/operations.d.ts.map +1 -1
- package/dist/constants/operations.js.map +1 -1
- package/dist/fs.d.ts +1 -1
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +0 -1
- package/dist/fs.js.map +1 -1
- package/dist/helpers/CSV.d.ts.map +1 -1
- package/dist/helpers/CSV.js +3 -1
- package/dist/helpers/CSV.js.map +1 -1
- package/dist/helpers/group-records.d.ts +12 -0
- package/dist/helpers/group-records.d.ts.map +1 -0
- package/dist/helpers/group-records.js +36 -0
- package/dist/helpers/group-records.js.map +1 -0
- package/dist/helpers/split-records.d.ts +30 -0
- package/dist/helpers/split-records.d.ts.map +1 -0
- package/dist/helpers/split-records.js +122 -0
- package/dist/helpers/split-records.js.map +1 -0
- package/package.json +4 -3
- package/dist/helpers/ByteDelimiterScanner.d.ts +0 -50
- package/dist/helpers/ByteDelimiterScanner.d.ts.map +0 -1
- package/dist/helpers/ByteDelimiterScanner.js +0 -103
- package/dist/helpers/ByteDelimiterScanner.js.map +0 -1
- package/dist/helpers/ByteRecordFilter.d.ts +0 -32
- package/dist/helpers/ByteRecordFilter.d.ts.map +0 -1
- package/dist/helpers/ByteRecordFilter.js +0 -89
- package/dist/helpers/ByteRecordFilter.js.map +0 -1
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ByteDelimiterScanner.d.ts","sourceRoot":"","sources":["../../src/helpers/ByteDelimiterScanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG;IAClC,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,IAAI,CAAC,EAAE,MAAM,CAAA;KACd,CAAA;CACF,CAAA;AAED,MAAM,MAAM,QAAQ,GAAG;IACrB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,GAAG,CAAC,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,2BAA2B,GAAG;IACxC,SAAS,EAAE,kBAAkB,CAAA;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,QAAQ,CAAA;IACf,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,eAAO,MAAM,wBAAwB,QAAY,CAAA;AAEjD,qBAAa,oBAAqB,SAAQ,SAAS;IACjD,SAAgB,MAAM,EAAE,SAAS,EAAE,CAAK;IAExC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAEvC,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,QAAQ,CAAI;IACpB,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,OAAO,CAAI;gBAEP,EACV,SAAS,EACT,KAAS,EACT,IAAI,EACJ,SAAgB,EAChB,QAAiB,EACjB,cAAyC,GAC1C,EAAE,2BAA2B;IAuCvB,UAAU,CACf,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,iBAAiB;IAStB,MAAM,CAAC,QAAQ,EAAE,iBAAiB;IAyBzC,OAAO,CAAC,IAAI;IAyCZ,OAAO,CAAC,GAAG;CAkBZ"}
|
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
import { Transform } from 'node:stream';
|
|
2
|
-
export const DEFAULT_INLINE_MAX_BYTES = 16 * 1024;
|
|
3
|
-
export class ByteDelimiterScanner extends Transform {
|
|
4
|
-
slices = [];
|
|
5
|
-
delimiter;
|
|
6
|
-
keepPrevious;
|
|
7
|
-
keepNext;
|
|
8
|
-
cutEvery;
|
|
9
|
-
trimStart;
|
|
10
|
-
trimEnd;
|
|
11
|
-
skipEmpty;
|
|
12
|
-
inlineMaxBytes;
|
|
13
|
-
pending = Buffer.alloc(0);
|
|
14
|
-
pendingAt = 0;
|
|
15
|
-
searchAt = 0;
|
|
16
|
-
sliceStart = 0;
|
|
17
|
-
matches = 0;
|
|
18
|
-
constructor({ delimiter, every = 1, trim, skipEmpty = true, encoding = 'utf8', inlineMaxBytes = DEFAULT_INLINE_MAX_BYTES, }) {
|
|
19
|
-
super();
|
|
20
|
-
this.delimiter = Buffer.from(delimiter.value, encoding);
|
|
21
|
-
this.keepPrevious = delimiter.keep?.previous ?? 0;
|
|
22
|
-
this.keepNext = delimiter.keep?.next ?? 0;
|
|
23
|
-
this.cutEvery = every;
|
|
24
|
-
this.trimStart = trim?.start ?? 0;
|
|
25
|
-
this.trimEnd = trim?.end ?? 0;
|
|
26
|
-
this.skipEmpty = skipEmpty;
|
|
27
|
-
this.inlineMaxBytes = inlineMaxBytes;
|
|
28
|
-
this.sliceStart = this.trimStart;
|
|
29
|
-
if (this.delimiter.length === 0)
|
|
30
|
-
throw new TypeError('Split delimiter must not be empty');
|
|
31
|
-
if (this.keepPrevious < 0 || this.keepNext < 0)
|
|
32
|
-
throw new TypeError('Split delimiter keep sizes must not be negative');
|
|
33
|
-
if (this.keepPrevious + this.keepNext > this.delimiter.length)
|
|
34
|
-
throw new TypeError('Split delimiter keep sizes must not exceed the delimiter length');
|
|
35
|
-
if (!Number.isInteger(every) || every < 1)
|
|
36
|
-
throw new TypeError('Split every must be a positive integer');
|
|
37
|
-
if (!Number.isInteger(this.trimStart) ||
|
|
38
|
-
!Number.isInteger(this.trimEnd) ||
|
|
39
|
-
this.trimStart < 0 ||
|
|
40
|
-
this.trimEnd < 0)
|
|
41
|
-
throw new TypeError('Split trim sizes must be non-negative integers');
|
|
42
|
-
if (!Number.isInteger(inlineMaxBytes) || inlineMaxBytes < 0)
|
|
43
|
-
throw new TypeError('Split inlineMaxBytes must be a non-negative integer');
|
|
44
|
-
}
|
|
45
|
-
_transform(chunk, _encoding, callback) {
|
|
46
|
-
this.scan(this.pending.length > 0 ? Buffer.concat([this.pending, chunk]) : chunk);
|
|
47
|
-
callback(null, chunk);
|
|
48
|
-
}
|
|
49
|
-
_flush(callback) {
|
|
50
|
-
const limit = this.pendingAt + this.pending.length - this.trimEnd;
|
|
51
|
-
while (true) {
|
|
52
|
-
const last = this.slices.at(-1);
|
|
53
|
-
if (!last || last.end <= limit)
|
|
54
|
-
break;
|
|
55
|
-
if (last.start >= limit) {
|
|
56
|
-
this.slices.pop();
|
|
57
|
-
continue;
|
|
58
|
-
}
|
|
59
|
-
last.end = limit;
|
|
60
|
-
if (last.content)
|
|
61
|
-
last.content = last.content.subarray(0, limit - last.start);
|
|
62
|
-
}
|
|
63
|
-
if (this.sliceStart <= limit)
|
|
64
|
-
this.cut(this.sliceStart, limit, this.pending);
|
|
65
|
-
callback();
|
|
66
|
-
}
|
|
67
|
-
scan(buffer) {
|
|
68
|
-
let searchFrom = Math.max(0, this.searchAt - this.pendingAt);
|
|
69
|
-
while (true) {
|
|
70
|
-
const index = buffer.indexOf(this.delimiter, searchFrom);
|
|
71
|
-
if (index === -1)
|
|
72
|
-
break;
|
|
73
|
-
const at = this.pendingAt + index;
|
|
74
|
-
searchFrom = index + this.delimiter.length;
|
|
75
|
-
if (at < this.trimStart)
|
|
76
|
-
continue;
|
|
77
|
-
this.matches += 1;
|
|
78
|
-
if (this.matches % this.cutEvery !== 0)
|
|
79
|
-
continue;
|
|
80
|
-
this.cut(this.sliceStart, at + this.keepPrevious, buffer);
|
|
81
|
-
this.sliceStart = at + this.delimiter.length - this.keepNext;
|
|
82
|
-
}
|
|
83
|
-
const tail = Math.max(searchFrom, buffer.length - (this.delimiter.length - 1));
|
|
84
|
-
this.searchAt = this.pendingAt + tail;
|
|
85
|
-
const openAt = this.sliceStart - this.pendingAt;
|
|
86
|
-
const isOpenSliceInline = openAt >= 0 && buffer.length - openAt <= this.inlineMaxBytes;
|
|
87
|
-
const keepFrom = isOpenSliceInline ? Math.min(tail, openAt) : tail;
|
|
88
|
-
this.pending = buffer.subarray(keepFrom);
|
|
89
|
-
this.pendingAt += keepFrom;
|
|
90
|
-
}
|
|
91
|
-
cut(start, end, buffer) {
|
|
92
|
-
if (this.skipEmpty && end <= start)
|
|
93
|
-
return;
|
|
94
|
-
const isInline = start >= this.pendingAt && end - start <= this.inlineMaxBytes;
|
|
95
|
-
if (!isInline) {
|
|
96
|
-
this.slices.push({ start, end });
|
|
97
|
-
return;
|
|
98
|
-
}
|
|
99
|
-
const content = Buffer.from(buffer.subarray(start - this.pendingAt, end - this.pendingAt));
|
|
100
|
-
this.slices.push({ start, end, content });
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
//# sourceMappingURL=ByteDelimiterScanner.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ByteDelimiterScanner.js","sourceRoot":"","sources":["../../src/helpers/ByteDelimiterScanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAA;AAiC/D,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAA;AAEjD,MAAM,OAAO,oBAAqB,SAAQ,SAAS;IACjC,MAAM,GAAgB,EAAE,CAAA;IAEvB,SAAS,CAAQ;IACjB,YAAY,CAAQ;IACpB,QAAQ,CAAQ;IAChB,QAAQ,CAAQ;IAChB,SAAS,CAAQ;IACjB,OAAO,CAAQ;IACf,SAAS,CAAS;IAClB,cAAc,CAAQ;IAE/B,OAAO,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACjC,SAAS,GAAG,CAAC,CAAA;IACb,QAAQ,GAAG,CAAC,CAAA;IACZ,UAAU,GAAG,CAAC,CAAA;IACd,OAAO,GAAG,CAAC,CAAA;IAEnB,YAAY,EACV,SAAS,EACT,KAAK,GAAG,CAAC,EACT,IAAI,EACJ,SAAS,GAAG,IAAI,EAChB,QAAQ,GAAG,MAAM,EACjB,cAAc,GAAG,wBAAwB,GACb;QAC5B,KAAK,EAAE,CAAA;QAEP,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACvD,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,QAAQ,IAAI,CAAC,CAAA;QACjD,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAA;QACzC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,CAAA;QAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,CAAA;QAEhC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAC7B,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAA;QAE1D,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC;YAC5C,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAA;QAExE,IAAI,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM;YAC3D,MAAM,IAAI,SAAS,CACjB,iEAAiE,CAClE,CAAA;QAEH,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;YACvC,MAAM,IAAI,SAAS,CAAC,wCAAwC,CAAC,CAAA;QAE/D,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC;YACjC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;YAC/B,IAAI,CAAC,SAAS,GAAG,CAAC;YAClB,IAAI,CAAC,OAAO,GAAG,CAAC;YAEhB,MAAM,IAAI,SAAS,CAAC,gDAAgD,CAAC,CAAA;QAEvE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC;YACzD,MAAM,IAAI,SAAS,CAAC,qDAAqD,CAAC,CAAA;IAC9E,CAAC;IAEM,UAAU,CACf,KAAa,EACb,SAAyB,EACzB,QAA2B;QAE3B,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CACvE,CAAA;QAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACvB,CAAC;IAEM,MAAM,CAAC,QAA2B;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAA;QAEjE,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,IAAI,KAAK;gBAAE,MAAK;YAErC,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAA;gBAEjB,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,GAAG,GAAG,KAAK,CAAA;YAEhB,IAAI,IAAI,CAAC,OAAO;gBACd,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/D,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,IAAI,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QAE5E,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,IAAI,CAAC,MAAc;QACzB,IAAI,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAA;QAE5D,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;YAExD,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,MAAK;YAEvB,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;YAEjC,UAAU,GAAG,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAA;YAE1C,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS;gBAAE,SAAQ;YAEjC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;YAEjB,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,KAAK,CAAC;gBAAE,SAAQ;YAEhD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;YAEzD,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAA;QAC9D,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CACnB,UAAU,EACV,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAC5C,CAAA;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QAErC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,CAAA;QAE/C,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,CAAC,cAAc,CAAA;QAE9D,MAAM,QAAQ,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAElE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACxC,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAA;IAC5B,CAAC;IAEO,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc;QACpD,IAAI,IAAI,CAAC,SAAS,IAAI,GAAG,IAAI,KAAK;YAAE,OAAM;QAE1C,MAAM,QAAQ,GACZ,KAAK,IAAI,IAAI,CAAC,SAAS,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,cAAc,CAAA;QAE/D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YAEhC,OAAM;QACR,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CACzB,MAAM,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAC9D,CAAA;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAA;IAC3C,CAAC;CACF"}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { Transform, type TransformCallback } from 'node:stream';
|
|
2
|
-
import type { ByteTrim, FileSplitDelimiter } from './ByteDelimiterScanner.js';
|
|
3
|
-
export type ByteRecordFilterOptions = {
|
|
4
|
-
delimiter: FileSplitDelimiter;
|
|
5
|
-
accept: (record: Buffer) => boolean;
|
|
6
|
-
trim?: ByteTrim;
|
|
7
|
-
encoding?: BufferEncoding;
|
|
8
|
-
maxRecordBytes?: number;
|
|
9
|
-
};
|
|
10
|
-
export declare const DEFAULT_FILTER_MAX_RECORD_BYTES: number;
|
|
11
|
-
export declare class ByteRecordFilter extends Transform {
|
|
12
|
-
accepted: number;
|
|
13
|
-
private readonly delimiter;
|
|
14
|
-
private readonly joint;
|
|
15
|
-
private readonly keepPrevious;
|
|
16
|
-
private readonly keepNext;
|
|
17
|
-
private readonly trimStart;
|
|
18
|
-
private readonly trimEnd;
|
|
19
|
-
private readonly maxRecordBytes;
|
|
20
|
-
private readonly accept;
|
|
21
|
-
private pending;
|
|
22
|
-
private pendingAt;
|
|
23
|
-
private received;
|
|
24
|
-
private searchAt;
|
|
25
|
-
private recordStart;
|
|
26
|
-
constructor({ delimiter, accept, trim, encoding, maxRecordBytes, }: ByteRecordFilterOptions);
|
|
27
|
-
_transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback): void;
|
|
28
|
-
_flush(callback: TransformCallback): void;
|
|
29
|
-
private scan;
|
|
30
|
-
private settle;
|
|
31
|
-
}
|
|
32
|
-
//# sourceMappingURL=ByteRecordFilter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ByteRecordFilter.d.ts","sourceRoot":"","sources":["../../src/helpers/ByteRecordFilter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAE/D,OAAO,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAE7E,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,EAAE,kBAAkB,CAAA;IAC7B,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAA;IACnC,IAAI,CAAC,EAAE,QAAQ,CAAA;IACf,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,eAAO,MAAM,+BAA+B,QAAc,CAAA;AAE1D,qBAAa,gBAAiB,SAAQ,SAAS;IACtC,QAAQ,SAAI;IAEnB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IAEpD,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,QAAQ,CAAI;IACpB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,WAAW,CAAQ;gBAEf,EACV,SAAS,EACT,MAAM,EACN,IAAI,EACJ,QAAiB,EACjB,cAAgD,GACjD,EAAE,uBAAuB;IAqBnB,UAAU,CACf,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,iBAAiB;IAiBtB,MAAM,CAAC,QAAQ,EAAE,iBAAiB;IAYzC,OAAO,CAAC,IAAI;IA8CZ,OAAO,CAAC,MAAM;CAcf"}
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
import { Transform } from 'node:stream';
|
|
2
|
-
export const DEFAULT_FILTER_MAX_RECORD_BYTES = 1024 * 1024;
|
|
3
|
-
export class ByteRecordFilter extends Transform {
|
|
4
|
-
accepted = 0;
|
|
5
|
-
delimiter;
|
|
6
|
-
joint;
|
|
7
|
-
keepPrevious;
|
|
8
|
-
keepNext;
|
|
9
|
-
trimStart;
|
|
10
|
-
trimEnd;
|
|
11
|
-
maxRecordBytes;
|
|
12
|
-
accept;
|
|
13
|
-
pending = Buffer.alloc(0);
|
|
14
|
-
pendingAt = 0;
|
|
15
|
-
received = 0;
|
|
16
|
-
searchAt;
|
|
17
|
-
recordStart;
|
|
18
|
-
constructor({ delimiter, accept, trim, encoding = 'utf8', maxRecordBytes = DEFAULT_FILTER_MAX_RECORD_BYTES, }) {
|
|
19
|
-
super();
|
|
20
|
-
this.delimiter = Buffer.from(delimiter.value, encoding);
|
|
21
|
-
this.keepPrevious = delimiter.keep?.previous ?? 0;
|
|
22
|
-
this.keepNext = delimiter.keep?.next ?? 0;
|
|
23
|
-
this.joint = this.delimiter.subarray(this.keepPrevious, this.delimiter.length - this.keepNext);
|
|
24
|
-
this.trimStart = trim?.start ?? 0;
|
|
25
|
-
this.trimEnd = trim?.end ?? 0;
|
|
26
|
-
this.maxRecordBytes = maxRecordBytes;
|
|
27
|
-
this.accept = accept;
|
|
28
|
-
this.searchAt = this.trimStart;
|
|
29
|
-
this.recordStart = this.trimStart;
|
|
30
|
-
if (this.delimiter.length === 0)
|
|
31
|
-
throw new TypeError('Split delimiter must not be empty');
|
|
32
|
-
}
|
|
33
|
-
_transform(chunk, _encoding, callback) {
|
|
34
|
-
this.pending =
|
|
35
|
-
this.pending.length > 0 ? Buffer.concat([this.pending, chunk]) : chunk;
|
|
36
|
-
this.received += chunk.length;
|
|
37
|
-
try {
|
|
38
|
-
const output = this.scan(this.received - this.trimEnd);
|
|
39
|
-
if (output.length > 0)
|
|
40
|
-
this.push(output);
|
|
41
|
-
callback();
|
|
42
|
-
}
|
|
43
|
-
catch (error) {
|
|
44
|
-
callback(error);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
_flush(callback) {
|
|
48
|
-
const limit = this.received - this.trimEnd;
|
|
49
|
-
const output = [];
|
|
50
|
-
if (this.recordStart <= limit)
|
|
51
|
-
this.settle(this.recordStart, limit, output);
|
|
52
|
-
if (output.length > 0)
|
|
53
|
-
this.push(Buffer.concat(output));
|
|
54
|
-
callback();
|
|
55
|
-
}
|
|
56
|
-
scan(limit) {
|
|
57
|
-
const output = [];
|
|
58
|
-
while (true) {
|
|
59
|
-
const index = this.pending.indexOf(this.delimiter, this.searchAt - this.pendingAt);
|
|
60
|
-
if (index === -1) {
|
|
61
|
-
this.searchAt = Math.max(this.searchAt, this.pendingAt + this.pending.length - (this.delimiter.length - 1));
|
|
62
|
-
break;
|
|
63
|
-
}
|
|
64
|
-
const at = this.pendingAt + index;
|
|
65
|
-
if (at + this.delimiter.length > limit)
|
|
66
|
-
break;
|
|
67
|
-
this.settle(this.recordStart, at + this.keepPrevious, output);
|
|
68
|
-
this.recordStart = at + this.delimiter.length - this.keepNext;
|
|
69
|
-
this.searchAt = at + this.delimiter.length;
|
|
70
|
-
}
|
|
71
|
-
const end = this.pendingAt + this.pending.length;
|
|
72
|
-
const keepFrom = Math.min(Math.max(Math.min(this.recordStart, this.searchAt), this.pendingAt), end);
|
|
73
|
-
this.pending = this.pending.subarray(keepFrom - this.pendingAt);
|
|
74
|
-
this.pendingAt = keepFrom;
|
|
75
|
-
if (end - this.recordStart > this.maxRecordBytes)
|
|
76
|
-
throw new RangeError(`Split where can only test records up to ${this.maxRecordBytes} bytes, found a record starting at byte ${this.recordStart} without a delimiter`);
|
|
77
|
-
return Buffer.concat(output);
|
|
78
|
-
}
|
|
79
|
-
settle(start, end, output) {
|
|
80
|
-
const record = this.pending.subarray(start - this.pendingAt, end - this.pendingAt);
|
|
81
|
-
if (!this.accept(record))
|
|
82
|
-
return;
|
|
83
|
-
if (this.accepted > 0)
|
|
84
|
-
output.push(this.joint);
|
|
85
|
-
output.push(record);
|
|
86
|
-
this.accepted += 1;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
//# sourceMappingURL=ByteRecordFilter.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ByteRecordFilter.js","sourceRoot":"","sources":["../../src/helpers/ByteRecordFilter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAA;AAY/D,MAAM,CAAC,MAAM,+BAA+B,GAAG,IAAI,GAAG,IAAI,CAAA;AAE1D,MAAM,OAAO,gBAAiB,SAAQ,SAAS;IACtC,QAAQ,GAAG,CAAC,CAAA;IAEF,SAAS,CAAQ;IACjB,KAAK,CAAQ;IACb,YAAY,CAAQ;IACpB,QAAQ,CAAQ;IAChB,SAAS,CAAQ;IACjB,OAAO,CAAQ;IACf,cAAc,CAAQ;IACtB,MAAM,CAA6B;IAE5C,OAAO,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACjC,SAAS,GAAG,CAAC,CAAA;IACb,QAAQ,GAAG,CAAC,CAAA;IACZ,QAAQ,CAAQ;IAChB,WAAW,CAAQ;IAE3B,YAAY,EACV,SAAS,EACT,MAAM,EACN,IAAI,EACJ,QAAQ,GAAG,MAAM,EACjB,cAAc,GAAG,+BAA+B,GACxB;QACxB,KAAK,EAAE,CAAA;QAEP,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACvD,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,QAAQ,IAAI,CAAC,CAAA;QACjD,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAA;QACzC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAClC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CACtC,CAAA;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,CAAA;QAC7B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAA;QAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAA;QAEjC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAC7B,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAA;IAC5D,CAAC;IAEM,UAAU,CACf,KAAa,EACb,SAAyB,EACzB,QAA2B;QAE3B,IAAI,CAAC,OAAO;YACV,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;QACxE,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;QAE7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;YAEtD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAExC,QAAQ,EAAE,CAAA;QACZ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,KAAc,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAEM,MAAM,CAAC,QAA2B;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAA;QAE1C,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,IAAI,IAAI,CAAC,WAAW,IAAI,KAAK;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAE3E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;QAEvD,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,IAAI,CAAC,KAAa;QACxB,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAChC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAC/B,CAAA;YAED,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CACtB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CACnE,CAAA;gBAED,MAAK;YACP,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;YAEjC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAK;YAE7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;YAE7D,IAAI,CAAC,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAA;YAC7D,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAA;QAC5C,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QAEhD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EACnE,GAAG,CACJ,CAAA;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAA;QAC/D,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QAEzB,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc;YAC9C,MAAM,IAAI,UAAU,CAClB,2CAA2C,IAAI,CAAC,cAAc,2CAA2C,IAAI,CAAC,WAAW,sBAAsB,CAChJ,CAAA;QAEH,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;IAEO,MAAM,CAAC,KAAa,EAAE,GAAW,EAAE,MAAgB;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAClC,KAAK,GAAG,IAAI,CAAC,SAAS,EACtB,GAAG,GAAG,IAAI,CAAC,SAAS,CACrB,CAAA;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,OAAM;QAEhC,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAE9C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEnB,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;IACpB,CAAC;CACF"}
|