bumparena 0.0.1 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/arena.ts +125 -62
- package/bench/arena.ts +75 -0
- package/bench/array.ts +61 -0
- package/bench/bench.md +38 -0
- package/bench/generateTestdata.ts +3 -0
- package/bench/init.ts +38 -0
- package/package.json +23 -2
- package/test/directAccess.ts +28 -0
- package/test/{test.ts → ptrs.ts} +18 -19
- package/test/reserve.ts +28 -0
- package/test/test.test.ts +12 -0
package/arena.ts
CHANGED
|
@@ -19,12 +19,6 @@ export interface ArenaHeaders {
|
|
|
19
19
|
header1: number
|
|
20
20
|
header2: number
|
|
21
21
|
}
|
|
22
|
-
const enum SHIFTOFFSETS {
|
|
23
|
-
BYTE_8 = 0, // No SHIFT needed
|
|
24
|
-
BYTE_16 = 1,
|
|
25
|
-
BYTE_32 = 2,
|
|
26
|
-
BYTE_64 = 3,
|
|
27
|
-
}
|
|
28
22
|
const enum HEADERS {
|
|
29
23
|
TOTAL_LENGTH_0_32 = 0,
|
|
30
24
|
PAYLOAD_LENGTH_0_32 = 1,
|
|
@@ -66,6 +60,25 @@ export class Arena {
|
|
|
66
60
|
}
|
|
67
61
|
this._emptySpots = new Uint32Array(currentOffset)
|
|
68
62
|
}
|
|
63
|
+
|
|
64
|
+
private _u(n: number): number {
|
|
65
|
+
return n >>> 0;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private _idx32(byteOffset: number): number {
|
|
69
|
+
return (byteOffset >>> 0) >> 2;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
private _makePtr(offset: number, gen: number | bigint): ArenaLocation {
|
|
73
|
+
const bOffset = BigInt(offset >>> 0);
|
|
74
|
+
const bGen = BigInt(gen ?? 0);
|
|
75
|
+
return ((bOffset << 32n) | (bGen & 0xFFFFFFFFn)) as ArenaLocation;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private _getOffset(ptr: ArenaLocation): number {
|
|
79
|
+
return Number(BigInt(ptr) >> 32n) >>> 0;
|
|
80
|
+
}
|
|
81
|
+
|
|
69
82
|
private _getBucketCount(bucketIdx: number): number {
|
|
70
83
|
return this._emptySpots[this._bucketOffsets[bucketIdx]!]!;
|
|
71
84
|
}
|
|
@@ -82,50 +95,63 @@ export class Arena {
|
|
|
82
95
|
this._emptySpots[this._bucketOffsets[bucketIdx]! + slotIdx + 1] = offset;
|
|
83
96
|
}
|
|
84
97
|
|
|
85
|
-
private _initBlock(start: number, dataLength: number,
|
|
86
|
-
const
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
this._view32[
|
|
91
|
-
|
|
92
|
-
this._view32[
|
|
98
|
+
private _initBlock(start: number, dataLength: number, headers: { header0: number, header1: number, header2: number }) {
|
|
99
|
+
const idx = this._idx32(start)
|
|
100
|
+
const { header0: h0, header1: h1, header2: h2 } = headers;
|
|
101
|
+
|
|
102
|
+
this._view32[idx + HEADERS.TOTAL_LENGTH_0_32] = this._u((dataLength + this.HEADER_SIZE_BYTES + this._allignMask) & ~this._allignMask);
|
|
103
|
+
this._view32[idx + HEADERS.PAYLOAD_LENGTH_0_32] = dataLength;
|
|
104
|
+
|
|
105
|
+
this._view32[idx + (HEADERS.DELETED_8 >> 2)] = ((h2 || 0) << 24) | ((h1 || 0) << 16) | ((h0 || 0) << 8);
|
|
106
|
+
this._view8[start + HEADERS.DELETED_8] = 0;
|
|
93
107
|
}
|
|
94
108
|
|
|
95
109
|
public alloc(data: Uint8Array, headers?: ArenaCustomHeaders): ArenaLocation {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
110
|
+
headers ||= { header0: 0, header1: 0, header2: 0 };
|
|
111
|
+
|
|
112
|
+
const needed = (data.byteLength + this.HEADER_SIZE_BYTES + this._allignMask) & ~this._allignMask;
|
|
113
|
+
const bucketIdx = (needed >> this._allignShift) - 1;
|
|
99
114
|
if (bucketIdx >= 0 && bucketIdx < this._bucketcount) {
|
|
100
115
|
const count = this._getBucketCount(bucketIdx);
|
|
101
116
|
if (count > 0) {
|
|
102
|
-
const recycledOffset = this._getBucketOffset(bucketIdx, count - 1);
|
|
117
|
+
const recycledOffset = this._u(this._getBucketOffset(bucketIdx, count - 1));
|
|
103
118
|
this._setBucketCount(bucketIdx, count - 1);
|
|
104
119
|
this._initBlock(recycledOffset, data.length, headers);
|
|
105
|
-
|
|
120
|
+
|
|
121
|
+
const gen = this._view32[this._idx32(recycledOffset) + HEADERS.GENERATION_BYTE_0_32]!;
|
|
122
|
+
return this._makePtr(recycledOffset, gen)
|
|
106
123
|
}
|
|
107
124
|
}
|
|
108
|
-
const start = this._offset
|
|
109
|
-
this._checkForSpace(
|
|
110
|
-
this._initBlock(start, data.byteLength, headers
|
|
125
|
+
const start = this._u(this._offset)
|
|
126
|
+
if (this._checkForSpace(needed)) this._resize()
|
|
127
|
+
this._initBlock(start, data.byteLength, headers)
|
|
128
|
+
|
|
111
129
|
this._view8.set(data, start + this.HEADER_SIZE_BYTES)
|
|
112
|
-
this._offset =
|
|
113
|
-
|
|
130
|
+
this._offset = this._u((start + needed) & ~this._allignMask);
|
|
131
|
+
|
|
132
|
+
const gen = this._view32[this._idx32(start) + HEADERS.GENERATION_BYTE_0_32]!
|
|
133
|
+
return this._makePtr(start, gen)
|
|
114
134
|
}
|
|
115
135
|
public read(location: ArenaLocation): Uint8Array | null {
|
|
116
|
-
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
if (
|
|
121
|
-
|
|
136
|
+
const { start, generation } = this.translate(location);
|
|
137
|
+
const idx = this._idx32(start)
|
|
138
|
+
const currgen = BigInt(this._view32[idx + HEADERS.GENERATION_BYTE_0_32]!)
|
|
139
|
+
|
|
140
|
+
if (generation !== currgen) return null
|
|
141
|
+
if (this._view8[start + HEADERS.DELETED_8] === 1) return null
|
|
142
|
+
|
|
143
|
+
const dataLength = this._view32[idx + HEADERS.PAYLOAD_LENGTH_0_32]!;
|
|
144
|
+
return this._view8.subarray(start + this.HEADER_SIZE_BYTES, start + this.HEADER_SIZE_BYTES + dataLength);
|
|
122
145
|
}
|
|
123
146
|
public free(location: ArenaLocation): ArenaLocation {
|
|
124
|
-
|
|
125
|
-
const
|
|
126
|
-
this._view32[
|
|
127
|
-
|
|
128
|
-
|
|
147
|
+
const { start, generation: _ } = this.translate(location)
|
|
148
|
+
const idx = this._idx32(start)
|
|
149
|
+
const currgen = this._view32[idx + HEADERS.GENERATION_BYTE_0_32]!
|
|
150
|
+
|
|
151
|
+
this._view32[idx + HEADERS.GENERATION_BYTE_0_32] = currgen + 1
|
|
152
|
+
this._view8[this._u(start) + HEADERS.DELETED_8] = 1
|
|
153
|
+
|
|
154
|
+
const totalBlockSize = this._view32[this._idx32(start) + HEADERS.TOTAL_LENGTH_0_32]!
|
|
129
155
|
const bucketIdx = (totalBlockSize >> this._allignShift) - 1
|
|
130
156
|
if (bucketIdx >= 0 && bucketIdx < this._bucketcount) {
|
|
131
157
|
const count = this._getBucketCount(bucketIdx)
|
|
@@ -135,11 +161,11 @@ export class Arena {
|
|
|
135
161
|
this._setBucketCount(bucketIdx, count + 1);
|
|
136
162
|
}
|
|
137
163
|
}
|
|
138
|
-
return
|
|
164
|
+
return this._makePtr(0, 0) as ArenaLocation
|
|
139
165
|
}
|
|
140
166
|
|
|
141
167
|
private _checkForSpace(size: number): boolean {
|
|
142
|
-
if (this._buffer.byteLength >= this._offset + size) return false
|
|
168
|
+
if (this._buffer.byteLength >= (this._offset + size)) return false
|
|
143
169
|
return true
|
|
144
170
|
}
|
|
145
171
|
|
|
@@ -149,45 +175,48 @@ export class Arena {
|
|
|
149
175
|
this._view8 = new Uint8Array(this._buffer)
|
|
150
176
|
this._view32 = new Uint32Array(this._buffer)
|
|
151
177
|
}
|
|
178
|
+
|
|
152
179
|
public size(): number {
|
|
153
180
|
return this._buffer.byteLength
|
|
154
181
|
}
|
|
182
|
+
|
|
155
183
|
public getBuffer(): Uint8Array {
|
|
156
|
-
return this._view8.subarray(0, this._offset)
|
|
184
|
+
return this._view8.subarray(0, this._u(this._offset))
|
|
157
185
|
}
|
|
158
186
|
public reserve(size: number): Uint8Array {
|
|
159
|
-
const start = this._offset
|
|
187
|
+
const start = this._u(this._offset)
|
|
160
188
|
this._checkForSpace(start + this.HEADER_SIZE_BYTES + size + this._allignMask & ~this._allignMask) && this._resize()
|
|
161
|
-
this._initBlock(start, size, {})
|
|
162
|
-
this._offset = this._offset + size + this.HEADER_SIZE_BYTES + this._allignMask & ~this._allignMask
|
|
189
|
+
this._initBlock(start, size, { header0: 0, header1: 0, header2: 0 })
|
|
190
|
+
this._offset = (this._offset + size + this.HEADER_SIZE_BYTES + this._allignMask & ~this._allignMask) >>> 0
|
|
163
191
|
return this._view8.subarray(start + this.HEADER_SIZE_BYTES, start + this.HEADER_SIZE_BYTES + size)
|
|
164
192
|
}
|
|
165
193
|
public translate(ptr: ArenaLocation) {
|
|
166
|
-
|
|
167
|
-
const generation =
|
|
194
|
+
const start = this._getOffset(ptr)
|
|
195
|
+
const generation = ptr & 0xFFFFFFFFn;
|
|
168
196
|
return { start, generation }
|
|
169
197
|
}
|
|
170
198
|
public readWithHeaders(ptr: ArenaLocation): Uint8Array | null {
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
let length = this._view32[
|
|
174
|
-
|
|
175
|
-
if (diff !== 0n) return null
|
|
199
|
+
const { start, generation } = this.translate(ptr)
|
|
200
|
+
let idx32 = this._idx32(start)
|
|
201
|
+
let length = this._view32[idx32 + HEADERS.PAYLOAD_LENGTH_0_32]!
|
|
202
|
+
if (BigInt(this._view32[idx32 + HEADERS.GENERATION_BYTE_0_32]!) !== generation) return null
|
|
176
203
|
return this._view8.subarray(start + 12, start + this.HEADER_SIZE_BYTES + Number(length))
|
|
177
204
|
}
|
|
178
205
|
public label(): Array<ArenaLocation> {
|
|
179
206
|
const ptrArray = new Array<ArenaLocation>()
|
|
180
207
|
const limit = this._offset;
|
|
181
208
|
let pos = 0
|
|
182
|
-
while (pos < limit) {
|
|
183
|
-
const totalLength = this._view32[pos
|
|
209
|
+
while (this._u(pos) < this._u(limit)) {
|
|
210
|
+
const totalLength = this._view32[this._idx32(pos)]!;
|
|
184
211
|
if (totalLength === 0) {
|
|
185
|
-
pos = (pos + 16) & ~15
|
|
212
|
+
pos = this._idx32((pos + 16) & ~15)
|
|
186
213
|
continue;
|
|
187
214
|
}
|
|
188
|
-
const isDeleted = this._view8[pos + HEADERS.DELETED_8] === 1
|
|
215
|
+
const isDeleted = this._view8[this._u(pos) + HEADERS.DELETED_8] === 1
|
|
189
216
|
if (!isDeleted) {
|
|
190
|
-
ptrArray.push(
|
|
217
|
+
ptrArray.push(
|
|
218
|
+
this._makePtr(pos, this._view32[this._idx32(pos) + HEADERS.GENERATION_BYTE_0_32]!)
|
|
219
|
+
);
|
|
191
220
|
}
|
|
192
221
|
pos += totalLength
|
|
193
222
|
if (totalLength === 0) break;
|
|
@@ -195,20 +224,54 @@ export class Arena {
|
|
|
195
224
|
return ptrArray
|
|
196
225
|
}
|
|
197
226
|
public getHeaders(ptr: ArenaLocation): ArenaHeaders {
|
|
198
|
-
|
|
227
|
+
const { start, generation: _ } = this.translate(ptr)
|
|
228
|
+
const idx = this._u(start)
|
|
199
229
|
return {
|
|
200
|
-
totalLength: Number(this._view32[(start
|
|
201
|
-
payloadlength: Number(this._view32[(start
|
|
202
|
-
deleted: this._view8[
|
|
203
|
-
header0: Number(this._view8[
|
|
204
|
-
header1: Number(this._view8[
|
|
205
|
-
header2: Number(this._view8[
|
|
230
|
+
totalLength: Number(this._view32[this._idx32(start) + HEADERS.TOTAL_LENGTH_0_32]!.toString()),
|
|
231
|
+
payloadlength: Number(this._view32[this._idx32(start) + HEADERS.PAYLOAD_LENGTH_0_32]!.toString()),
|
|
232
|
+
deleted: this._view8[idx + HEADERS.DELETED_8] === 1,
|
|
233
|
+
header0: Number(this._view8[idx + HEADERS.USER_STATUS_0_8]!.toString()),
|
|
234
|
+
header1: Number(this._view8[idx + HEADERS.USER_STATUS_1_8]!.toString()),
|
|
235
|
+
header2: Number(this._view8[idx + HEADERS.USER_STATUS_2_8]!.toString()),
|
|
206
236
|
}
|
|
207
237
|
}
|
|
208
238
|
public estimate(size: number, amnt: number): number {
|
|
209
239
|
return (((size + this._allignMask) & ~this._allignMask) + this.HEADER_SIZE_BYTES) * amnt
|
|
210
240
|
}
|
|
211
|
-
|
|
212
|
-
|
|
241
|
+
public directAlloc(source: Uint8Array, startn: number, endn: number) {
|
|
242
|
+
const start = this._u(this._offset)
|
|
243
|
+
const len = endn - startn;
|
|
244
|
+
const needed = (len + this.HEADER_SIZE_BYTES + this._allignMask) & ~this._allignMask;
|
|
245
|
+
const bucketIdx = (needed >> this._allignShift) - 1;
|
|
246
|
+
if (bucketIdx >= 0 && bucketIdx < this._bucketcount) {
|
|
247
|
+
const count = this._getBucketCount(bucketIdx);
|
|
248
|
+
if (count > 0) {
|
|
249
|
+
const recycledOffset = this._u(this._getBucketOffset(bucketIdx, count - 1));
|
|
250
|
+
this._setBucketCount(bucketIdx, count - 1);
|
|
251
|
+
this._initBlock(recycledOffset, needed, { header0: 0, header1: 0, header2: 0 });
|
|
252
|
+
const gen = this._view32[this._idx32(recycledOffset) + HEADERS.GENERATION_BYTE_0_32]!;
|
|
253
|
+
this._view8.set(source.subarray(startn, endn), recycledOffset + this.HEADER_SIZE_BYTES)
|
|
254
|
+
return this._makePtr(recycledOffset, gen)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (this._checkForSpace(needed)) this._resize()
|
|
258
|
+
this._initBlock(start, len, { header0: 0, header1: 0, header2: 0 });
|
|
259
|
+
this._offset = this._u((start + needed) & ~this._allignMask);
|
|
260
|
+
this._view8.set(source.subarray(startn, endn), start + this.HEADER_SIZE_BYTES)
|
|
261
|
+
const gen = this._view32[this._idx32(start) + HEADERS.GENERATION_BYTE_0_32]!
|
|
262
|
+
return this._makePtr(start, gen)
|
|
213
263
|
|
|
264
|
+
// if (this._checkForSpace(needed)) this._resize()
|
|
265
|
+
// this._initBlock(start, needed, { header0: 0, header1: 0, header2: 0 })
|
|
266
|
+
// this._view8.set(source.subarray(startn, endn), start + this.HEADER_SIZE_BYTES)
|
|
267
|
+
// this._offset = this._u((start + needed) & ~this._allignMask);
|
|
268
|
+
//
|
|
269
|
+
// const gen = this._view32[this._idx32(start) + HEADERS.GENERATION_BYTE_0_32]!
|
|
270
|
+
// return this._makePtr(start, gen)
|
|
214
271
|
|
|
272
|
+
}
|
|
273
|
+
public clear() {
|
|
274
|
+
this._offset = 0
|
|
275
|
+
this._emptySpots.fill(0)
|
|
276
|
+
}
|
|
277
|
+
}
|
package/bench/arena.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { pipeline } from "node:stream/promises";
|
|
2
|
+
import { Arena } from "../arena";
|
|
3
|
+
import { stream } from "./init.ts"
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
let tmp: Uint8Array | null = null
|
|
7
|
+
function getNewArenaTransform() {
|
|
8
|
+
return new WritableStream<Uint8Array>({
|
|
9
|
+
write(chunk) {
|
|
10
|
+
let sourceIdx = 0;
|
|
11
|
+
const chunkLen = chunk.byteLength;
|
|
12
|
+
|
|
13
|
+
if (tmp) {
|
|
14
|
+
const nextNewline = chunk.indexOf(10);
|
|
15
|
+
if (nextNewline !== -1) {
|
|
16
|
+
const totalLen = tmp.byteLength + nextNewline;
|
|
17
|
+
const target = new Uint8Array(totalLen)
|
|
18
|
+
target.set(tmp, 0);
|
|
19
|
+
target.set(chunk.subarray(0, nextNewline), tmp.byteLength);
|
|
20
|
+
arena.directAlloc(target, 0, target.length)
|
|
21
|
+
sourceIdx = nextNewline + 1;
|
|
22
|
+
tmp = null;
|
|
23
|
+
} else {
|
|
24
|
+
const newTmp = new Uint8Array(tmp.byteLength + chunkLen);
|
|
25
|
+
newTmp.set(tmp);
|
|
26
|
+
newTmp.set(chunk);
|
|
27
|
+
tmp = newTmp;
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// 2. Schnelle Schleife für den Rest des Chunks
|
|
33
|
+
while (true) {
|
|
34
|
+
const idx = chunk.indexOf(10, sourceIdx);
|
|
35
|
+
if (idx === -1) break;
|
|
36
|
+
|
|
37
|
+
const lineLength = idx - sourceIdx;
|
|
38
|
+
if (lineLength > 0) {
|
|
39
|
+
arena.directAlloc(chunk, 0, lineLength);
|
|
40
|
+
}
|
|
41
|
+
sourceIdx = idx + 1;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (sourceIdx < chunkLen) {
|
|
45
|
+
tmp = chunk.slice(sourceIdx); // slice kopiert hier einmalig den Rest
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
close() {
|
|
49
|
+
if (tmp) {
|
|
50
|
+
arena.directAlloc(tmp, 0, tmp.length)
|
|
51
|
+
tmp = null
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
})
|
|
55
|
+
};
|
|
56
|
+
const arena = new Arena();
|
|
57
|
+
(async () => {
|
|
58
|
+
const start = performance.now()
|
|
59
|
+
for (let i = 0; i < 10; i++) {
|
|
60
|
+
tmp = null
|
|
61
|
+
const arenatransform = getNewArenaTransform()
|
|
62
|
+
const reader = stream()
|
|
63
|
+
|
|
64
|
+
await pipeline(
|
|
65
|
+
reader,
|
|
66
|
+
arenatransform
|
|
67
|
+
)
|
|
68
|
+
const _ptr = arena.label()
|
|
69
|
+
arena.clear()
|
|
70
|
+
}
|
|
71
|
+
const end = performance.now()
|
|
72
|
+
console.log(`${(end - start).toFixed(2)} ms`)
|
|
73
|
+
console.log(process.memoryUsage())
|
|
74
|
+
console.log(`arenasize: ${arena.size() / 1024} KB`)
|
|
75
|
+
})();
|
package/bench/array.ts
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { pipeline } from "node:stream/promises";
|
|
2
|
+
import { noopWriter, stream } from "./init.ts"
|
|
3
|
+
|
|
4
|
+
const array: Array<string> = [];
|
|
5
|
+
const decode = new TextDecoder()
|
|
6
|
+
let tmp: Uint8Array | null = null // Rest vom vorherigen Chunk
|
|
7
|
+
function getNewArrayTransformStream() {
|
|
8
|
+
return new WritableStream<Uint8Array>({
|
|
9
|
+
write(chunk) {
|
|
10
|
+
if (tmp) {
|
|
11
|
+
const combined = new Uint8Array(tmp.byteLength + chunk.byteLength)
|
|
12
|
+
combined.set(tmp, 0)
|
|
13
|
+
combined.set(chunk, tmp.byteLength)
|
|
14
|
+
chunk = combined
|
|
15
|
+
tmp = null
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let firstIdx = 0
|
|
19
|
+
let idx: number
|
|
20
|
+
|
|
21
|
+
while ((idx = chunk.indexOf(10, firstIdx)) !== -1) {
|
|
22
|
+
const lineLength = idx - firstIdx
|
|
23
|
+
if (lineLength > 0) {
|
|
24
|
+
array.push(decode.decode(chunk.subarray(firstIdx, idx)!)!)
|
|
25
|
+
}
|
|
26
|
+
firstIdx = idx + 1
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (firstIdx < chunk.byteLength) {
|
|
30
|
+
tmp = chunk.subarray(firstIdx)
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
close() {
|
|
34
|
+
if (tmp) {
|
|
35
|
+
//@ts-ignore
|
|
36
|
+
array.push(tmp)
|
|
37
|
+
tmp = null
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
})
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
await (async () => {
|
|
45
|
+
const start = performance.now()
|
|
46
|
+
for (let i = 0; i < 100; i++) {
|
|
47
|
+
array.length = 0
|
|
48
|
+
tmp = null
|
|
49
|
+
const arraytransform = getNewArrayTransformStream()
|
|
50
|
+
const reader = stream()
|
|
51
|
+
await pipeline(
|
|
52
|
+
reader,
|
|
53
|
+
arraytransform,
|
|
54
|
+
)
|
|
55
|
+
for (const iter of array) noopWriter(iter)
|
|
56
|
+
}
|
|
57
|
+
const end = performance.now()
|
|
58
|
+
console.log(`${(end - start).toFixed(2)} ms`)
|
|
59
|
+
console.log(`Count: ${array.length}`)
|
|
60
|
+
console.log(process.memoryUsage())
|
|
61
|
+
})();
|
package/bench/bench.md
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Benchmark 1
|
|
2
|
+
|
|
3
|
+
## bun arena.ts;
|
|
4
|
+
183098.55 ms
|
|
5
|
+
{
|
|
6
|
+
rss: 11240919040,
|
|
7
|
+
heapTotal: 8595075072,
|
|
8
|
+
heapUsed: 4817152633,
|
|
9
|
+
external: 2157456969,
|
|
10
|
+
arrayBuffers: 2156686216,
|
|
11
|
+
}
|
|
12
|
+
arenasize: 2097152 KB
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## bun array.ts
|
|
17
|
+
765961.27 ms
|
|
18
|
+
Count: 50000000
|
|
19
|
+
{
|
|
20
|
+
rss: 28036014080,
|
|
21
|
+
heapTotal: 9476152320,
|
|
22
|
+
heapUsed: 11222438346,
|
|
23
|
+
external: 7170077276,
|
|
24
|
+
arrayBuffers: 2921230601,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Direct Comparison
|
|
30
|
+
|
|
31
|
+
| Metric | Arena Implementation (Optimized) | Standard Array Implementation | Difference / Factor |
|
|
32
|
+
| :--- | :--- | :--- | :--- |
|
|
33
|
+
| **Total Time** | **183,098 ms** (3.05 min) | **765,961 ms** (12.76 min) | **~4.2x Faster** |
|
|
34
|
+
| **Time per 50M Lines** | **~18.3 sec** | **~76.6 sec** | **- 58.3 sec / Round** |
|
|
35
|
+
| **RAM Usage (RSS)** | **11.24 GB** | **28.04 GB** | **16.8 GB Saved** |
|
|
36
|
+
| **Heap Used (JS Objects)** | **4.82 GB** | **11.22 GB** | **2.3x More Efficient** |
|
|
37
|
+
| **External (Buffer)** | **2.16 GB** | **7.17 GB** | **Compact Memory Footprint** |
|
|
38
|
+
|
package/bench/init.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { createWriteStream, createReadStream } from "node:fs";
|
|
2
|
+
const testfile = "test3.txt"
|
|
3
|
+
function splitmix32(a: number) {
|
|
4
|
+
return function() {
|
|
5
|
+
a |= 0; a = a + 0x9e3779b9 | 0;
|
|
6
|
+
let t = a ^ a >>> 16; t = Math.imul(t, 0x21f0aaad);
|
|
7
|
+
t = t ^ t >>> 15; t = Math.imul(t, 0x735a2d97);
|
|
8
|
+
return ((t = t ^ t >>> 15) >>> 0) / 4294967296;
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export function noopWriter(_data: any) { return true; };
|
|
13
|
+
|
|
14
|
+
async function generateTestData(seed: number, count: number) {
|
|
15
|
+
const writer = createWriteStream(testfile, { highWaterMark: 16 * 1024 * 1024 })
|
|
16
|
+
const random = splitmix32(seed)
|
|
17
|
+
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
|
|
18
|
+
for (let i = 0; i < count; i++) {
|
|
19
|
+
let str = '';
|
|
20
|
+
const len = Math.floor(random() * 5) + 10;
|
|
21
|
+
for (let j = 0; j < len; j++) {
|
|
22
|
+
str += characters.charAt(Math.floor(Math.random() * characters.length));
|
|
23
|
+
}
|
|
24
|
+
if (!writer.write(str + "\n")) {
|
|
25
|
+
await new Promise((res, _rej) => {
|
|
26
|
+
writer.once("drain", res)
|
|
27
|
+
})
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export const stream = () => {
|
|
33
|
+
return createReadStream(testfile)
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export function generateData(seed: number, count: number) {
|
|
37
|
+
generateTestData(seed, count);
|
|
38
|
+
}
|
package/package.json
CHANGED
|
@@ -1,16 +1,37 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bumparena",
|
|
3
|
-
"version":"0.0.
|
|
3
|
+
"version":"0.0.5",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/arena.js",
|
|
7
7
|
"module": "dist/arena.js",
|
|
8
8
|
"types": "dist/arena.d.ts",
|
|
9
|
+
"license":"MIT",
|
|
9
10
|
"scripts": {
|
|
10
11
|
"build": "tsc",
|
|
11
12
|
"test": "node ./dist/arena.js"
|
|
12
13
|
},
|
|
14
|
+
"repository": {
|
|
15
|
+
"type": "git",
|
|
16
|
+
"url": "git+https://github.com/eugen252009/bumparena.git"
|
|
17
|
+
},
|
|
13
18
|
"devDependencies": {
|
|
14
19
|
"@types/node": "^25.2.3"
|
|
15
|
-
}
|
|
20
|
+
},
|
|
21
|
+
"keywords": [
|
|
22
|
+
"arena",
|
|
23
|
+
"typescript",
|
|
24
|
+
"javascript",
|
|
25
|
+
"simulation",
|
|
26
|
+
"game-engine",
|
|
27
|
+
"algorithm",
|
|
28
|
+
"data-structure",
|
|
29
|
+
"buffer",
|
|
30
|
+
"uint8array",
|
|
31
|
+
"memory",
|
|
32
|
+
"testing",
|
|
33
|
+
"cjs",
|
|
34
|
+
"esm",
|
|
35
|
+
"library"
|
|
36
|
+
]
|
|
16
37
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Arena } from "../arena"
|
|
2
|
+
|
|
3
|
+
export function TestDirectAlloc(): boolean {
|
|
4
|
+
//Init
|
|
5
|
+
const count = 255
|
|
6
|
+
const a = new Arena()
|
|
7
|
+
const testdata = new Array<Uint8Array>()
|
|
8
|
+
for (let i = 0; i < count; i++) {
|
|
9
|
+
const data = new Uint8Array([i, 1, 2, 3, 4, 5, 6, 7])
|
|
10
|
+
testdata.push(data)
|
|
11
|
+
|
|
12
|
+
a.directAlloc(data, 0, data.length)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const ptrs = a.label()
|
|
16
|
+
if (ptrs.length !== count) throw new Error(`Allocated wrong amount of items!: needed: ${count}, got: ${ptrs.length}`)
|
|
17
|
+
|
|
18
|
+
for (let i = 0; i < ptrs.length; i++) {
|
|
19
|
+
const buf = a.read(ptrs[i])
|
|
20
|
+
if (!buf) throw new Error(`data at index ${i} is Empty.`)
|
|
21
|
+
|
|
22
|
+
for (let j = 0; j < testdata.length; j++) {
|
|
23
|
+
if (buf[j] !== testdata[i][j]) throw new Error(`dataset ${i} at ${j} is broken!`)
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return true
|
|
27
|
+
}
|
|
28
|
+
|
package/test/{test.ts → ptrs.ts}
RENAMED
|
@@ -1,18 +1,12 @@
|
|
|
1
|
-
import { Arena
|
|
1
|
+
import { Arena } from "../arena"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
function
|
|
5
|
-
|
|
3
|
+
|
|
4
|
+
export function TestCheckPtrAccess(): boolean {
|
|
5
|
+
//Init
|
|
6
6
|
const a = new Arena()
|
|
7
7
|
const testdata = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7])
|
|
8
8
|
const headers = { header0: 42, header1: 24, header2: 240 };
|
|
9
|
-
_testCheckPtrAccess(a, testdata, headers)
|
|
10
|
-
}
|
|
11
9
|
|
|
12
|
-
function _testCheckPtrAccess(a: Arena, testdata: Uint8Array, headers: ArenaCustomHeaders) {
|
|
13
|
-
// USE-AFTER-FREE
|
|
14
|
-
const useAfterFreePtr = a.alloc(testdata, headers)
|
|
15
|
-
a.free(useAfterFreePtr)
|
|
16
10
|
// free Slot reuse
|
|
17
11
|
const ptr = a.alloc(testdata, headers)
|
|
18
12
|
const res = a.read(ptr)
|
|
@@ -24,15 +18,20 @@ function _testCheckPtrAccess(a: Arena, testdata: Uint8Array, headers: ArenaCusto
|
|
|
24
18
|
if (resultheaders.header0 !== headers.header0) throw new Error(`header0: got: ${resultheaders.header0} needed: ${headers.header0}`)
|
|
25
19
|
if (resultheaders.header1 !== headers.header1) throw new Error(`header1: got: ${resultheaders.header1} needed: ${headers.header1}`)
|
|
26
20
|
if (resultheaders.header2 !== headers.header2) throw new Error(`header2: got: ${resultheaders.header2} needed: ${headers.header2}`)
|
|
21
|
+
if (res.toString() !== testdata.toString()) throw new Error(`${testdata} had a problem with data integrety`)
|
|
27
22
|
|
|
28
|
-
|
|
29
|
-
throw new Error(`${testdata} had a problem with data integrety`)
|
|
30
|
-
}
|
|
31
|
-
//final use after free with Generation this should never be possible
|
|
32
|
-
if (a.read(useAfterFreePtr) !== null) {
|
|
33
|
-
throw new Error(`Use after Free is Possible, this is a big Problem`)
|
|
34
|
-
}
|
|
35
|
-
console.log("CheckPtrAccess Done!")
|
|
23
|
+
return true
|
|
36
24
|
}
|
|
25
|
+
export function TestUseAfterFree(): boolean {
|
|
26
|
+
//Init
|
|
27
|
+
const a = new Arena()
|
|
28
|
+
const testdata = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7])
|
|
37
29
|
|
|
38
|
-
|
|
30
|
+
// USE-AFTER-FREE init
|
|
31
|
+
const useAfterFreePtr = a.alloc(testdata)
|
|
32
|
+
a.free(useAfterFreePtr)
|
|
33
|
+
//final use after free with Generation this should never be possible
|
|
34
|
+
if (a.read(useAfterFreePtr) !== null) throw new Error(`Use after Free is Possible, this is a big Problem`)
|
|
35
|
+
|
|
36
|
+
return true
|
|
37
|
+
}
|
package/test/reserve.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Arena } from "../arena"
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
export function TestCheckReserve(): boolean {
|
|
5
|
+
//Init
|
|
6
|
+
const count = 1_000
|
|
7
|
+
const a = new Arena()
|
|
8
|
+
const testdata = new Array<Uint8Array>()
|
|
9
|
+
for (let i = 0; i < count; i++) {
|
|
10
|
+
const data = new Uint8Array([i, 1, 2, 3, 4, 5, 6, 7])
|
|
11
|
+
testdata.push(data)
|
|
12
|
+
|
|
13
|
+
const buf = a.reserve(data.byteLength)
|
|
14
|
+
buf.set(data)
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const ptrs = a.label()
|
|
18
|
+
|
|
19
|
+
for (let i = 0; i < ptrs.length; i++) {
|
|
20
|
+
const buf = a.read(ptrs[i])
|
|
21
|
+
if (!buf) throw new Error(`reservation failed at item: ${i}`)
|
|
22
|
+
for (let j = 0; j < ptrs.length; j++) {
|
|
23
|
+
if (buf[j] !== testdata[i][j]) throw new Error(`reservation failed at item ${i}:${j}`)
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
return true
|
|
28
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { describe, test } from "node:test"
|
|
2
|
+
import { equal } from "node:assert"
|
|
3
|
+
import { TestCheckPtrAccess, TestUseAfterFree } from "./ptrs.ts"
|
|
4
|
+
import { TestCheckReserve } from "./reserve.ts"
|
|
5
|
+
import { TestDirectAlloc } from "./directAccess.ts"
|
|
6
|
+
|
|
7
|
+
describe("Arena Checks", () => {
|
|
8
|
+
test("Pointer Access", () => equal(TestCheckPtrAccess(), true, "Data corruption is possible!"))
|
|
9
|
+
test("Use after free", () => equal(TestUseAfterFree(), true, "Use after free is possible!"))
|
|
10
|
+
test("Test Reserving Data", () => equal(TestCheckReserve(), true, "Something with the reservation is not working properly"))
|
|
11
|
+
test("Test allocating data directly", () => equal(TestDirectAlloc(), true, "Direct alloc isnt working properly"))
|
|
12
|
+
})
|