@minnowdb/core 0.6.9 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/block-format/block.js +254 -325
- package/dist/block-format/checksum.js +30 -52
- package/dist/block-format/codecs.js +112 -125
- package/dist/block-format/column.js +187 -205
- package/dist/block-format/index.js +5 -1
- package/dist/block-format/physical.js +371 -387
- package/dist/block-format/types.js +4 -1
- package/dist/block-format/unicode.js +30 -39
- package/dist/date-value.d.ts +5 -0
- package/dist/date-value.js +71 -41
- package/dist/engine/artifact-cache.js +106 -116
- package/dist/engine/batch.js +36 -44
- package/dist/engine/buffered-writer.js +156 -158
- package/dist/engine/byte-estimates.js +20 -22
- package/dist/engine/cache-limits.js +24 -25
- package/dist/engine/cancellation.js +5 -3
- package/dist/engine/catalog.js +64 -73
- package/dist/engine/client.js +715 -780
- package/dist/engine/database.js +15952 -18465
- package/dist/engine/defaults.js +104 -111
- package/dist/engine/errors.js +113 -128
- package/dist/engine/fts.js +233 -280
- package/dist/engine/group-index.js +287 -321
- package/dist/engine/index.js +23 -12
- package/dist/engine/join-index.js +177 -186
- package/dist/engine/keyed-live.js +168 -173
- package/dist/engine/live-api.js +0 -1
- package/dist/engine/live-equal.js +32 -37
- package/dist/engine/live.js +603 -647
- package/dist/engine/memory.js +109 -111
- package/dist/engine/optimizer.d.ts +19 -1
- package/dist/engine/optimizer.js +2642 -2515
- package/dist/engine/point-read.d.ts +4 -2
- package/dist/engine/point-read.js +121 -171
- package/dist/engine/query-api.js +12 -3
- package/dist/engine/query-cache.js +61 -69
- package/dist/engine/query.d.ts +45 -0
- package/dist/engine/query.js +8506 -9375
- package/dist/engine/result-wire.js +221 -252
- package/dist/engine/schema-wire.js +97 -112
- package/dist/engine/schema.js +1099 -1254
- package/dist/engine/sort-keys.js +315 -389
- package/dist/engine/sql-domains.d.ts +7 -1
- package/dist/engine/sql-domains.js +648 -734
- package/dist/engine/sql-driver.js +0 -1
- package/dist/engine/sql-functions.d.ts +11 -0
- package/dist/engine/sql-functions.js +1092 -0
- package/dist/engine/sql-json.js +199 -218
- package/dist/engine/sql-semantics.d.ts +25 -4
- package/dist/engine/sql-semantics.js +579 -513
- package/dist/engine/typed-live.js +271 -293
- package/dist/engine/vector.d.ts +7 -1
- package/dist/engine/vector.js +4513 -4580
- package/dist/engine/worker-host.d.ts +8 -63
- package/dist/engine/worker-host.js +28 -1130
- package/dist/engine/worker-indexeddb.d.ts +1 -0
- package/dist/engine/worker-indexeddb.js +3 -0
- package/dist/engine/worker-memory.d.ts +1 -0
- package/dist/engine/worker-memory.js +3 -0
- package/dist/engine/worker-opfs.d.ts +1 -0
- package/dist/engine/worker-opfs.js +3 -0
- package/dist/engine/worker-server.d.ts +93 -0
- package/dist/engine/worker-server.js +999 -0
- package/dist/engine/worker-store-indexeddb.d.ts +2 -0
- package/dist/engine/worker-store-indexeddb.js +11 -0
- package/dist/engine/worker-store-memory.d.ts +2 -0
- package/dist/engine/worker-store-memory.js +6 -0
- package/dist/engine/worker-store-opfs.d.ts +2 -0
- package/dist/engine/worker-store-opfs.js +9 -0
- package/dist/engine/worker.js +0 -12
- package/dist/engine/write-block-planner.js +93 -100
- package/dist/plan/index.js +15 -16
- package/dist/plan/model.d.ts +9 -1
- package/dist/plan/model.js +20 -27
- package/dist/storage/index.js +0 -12
- package/dist/storage/indexeddb.js +11776 -13171
- package/dist/storage/memory.js +1198 -1241
- package/dist/storage/opfs/files.js +238 -295
- package/dist/storage/opfs/index.js +9 -3
- package/dist/storage/opfs/leader.js +4386 -4856
- package/dist/storage/opfs/rpc.js +223 -259
- package/dist/storage/opfs/snapshot-ledger.js +219 -260
- package/dist/storage/opfs/store.js +884 -1003
- package/dist/storage/persistence.js +59 -74
- package/dist/storage/snapshot-stream.js +788 -826
- package/dist/storage/snapshot.js +36 -37
- package/dist/storage/toolkit/extents.js +444 -520
- package/dist/storage/toolkit/index.js +28 -29
- package/dist/storage/toolkit/record-core.js +5792 -6377
- package/dist/storage/toolkit/sync-file.js +34 -42
- package/dist/storage/toolkit/wal.js +92 -127
- package/dist/storage/toolkit/wire.js +232 -256
- package/dist/storage/types.js +2952 -3228
- package/dist/testing/block-store-conformance.js +1556 -1629
- package/dist/testing/index.js +310 -291
- package/dist/testing/opfs-shim.js +280 -312
- package/dist/testing/simulator.js +496 -534
- package/dist/testing/sqllogictest.js +425 -365
- package/dist/transactions/index.js +1464 -1712
- package/dist/worker-protocol/index.js +70 -69
- package/package.json +20 -2
- package/postgres-feature-profile.json +15 -5
- package/sql-feature-matrix.json +252 -2
|
@@ -1,447 +1,431 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const MAX_DATETIME_MILLISECONDS = 8_640_000_000_000_000;
|
|
8
|
-
export function physicalBitmapByteLength(rowCount) {
|
|
9
|
-
assertRowCount(rowCount);
|
|
10
|
-
return Math.ceil(rowCount / 8);
|
|
1
|
+
const MAX_PHYSICAL_COLUMN_BYTE_LENGTH = 64 * 1024 * 1024;
|
|
2
|
+
const MAX_BLOCK_ROW_COUNT = 1048576;
|
|
3
|
+
const MAX_DATETIME_MILLISECONDS = 864e13;
|
|
4
|
+
function physicalBitmapByteLength(rowCount) {
|
|
5
|
+
assertRowCount(rowCount);
|
|
6
|
+
return Math.ceil(rowCount / 8);
|
|
11
7
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return byteLength;
|
|
8
|
+
function physicalColumnByteLength(type, rowCount, stringContentByteLength = 0) {
|
|
9
|
+
assertLogicalType(type);
|
|
10
|
+
const validityByteLength = physicalBitmapByteLength(rowCount);
|
|
11
|
+
assertNonnegativeSafeInteger(stringContentByteLength, "string content byte length");
|
|
12
|
+
let byteLength;
|
|
13
|
+
switch (type) {
|
|
14
|
+
case "boolean":
|
|
15
|
+
if (stringContentByteLength !== 0)
|
|
16
|
+
throw new RangeError("Boolean columns have no string content");
|
|
17
|
+
byteLength = checkedAdd(validityByteLength, validityByteLength, "physical column");
|
|
18
|
+
break;
|
|
19
|
+
case "number":
|
|
20
|
+
case "datetime":
|
|
21
|
+
if (stringContentByteLength !== 0)
|
|
22
|
+
throw new RangeError(`${type} columns have no string content`);
|
|
23
|
+
byteLength = checkedAdd(validityByteLength, checkedMultiply(rowCount, 8, "physical column"), "physical column");
|
|
24
|
+
break;
|
|
25
|
+
case "string":
|
|
26
|
+
byteLength = checkedAdd(validityByteLength, checkedAdd(checkedMultiply(checkedAdd(rowCount, 1, "string offsets"), 4, "string offsets"), stringContentByteLength, "physical column"), "physical column");
|
|
27
|
+
break;
|
|
28
|
+
}
|
|
29
|
+
assertPhysicalByteLength(byteLength);
|
|
30
|
+
return byteLength;
|
|
36
31
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
32
|
+
function validatePhysicalColumn(input) {
|
|
33
|
+
assertLogicalType(input.type);
|
|
34
|
+
assertRowCount(input.rowCount);
|
|
35
|
+
assertPhysicalByteLength(input.bytes.byteLength);
|
|
36
|
+
const validityByteLength = physicalBitmapByteLength(input.rowCount);
|
|
37
|
+
if (input.bytes.byteLength < validityByteLength)
|
|
38
|
+
throw new Error("Truncated validity bitmap");
|
|
39
|
+
const validity = input.bytes.subarray(0, validityByteLength);
|
|
40
|
+
assertZeroPaddingBits(validity, input.rowCount, "validity bitmap");
|
|
41
|
+
const nullCount = input.rowCount - countSetBits(validity, input.rowCount);
|
|
42
|
+
let metadata = {};
|
|
43
|
+
switch (input.type) {
|
|
44
|
+
case "boolean": {
|
|
45
|
+
const expectedLength = physicalColumnByteLength(input.type, input.rowCount);
|
|
46
|
+
if (input.bytes.byteLength !== expectedLength)
|
|
47
|
+
throw new Error("Invalid boolean payload");
|
|
48
|
+
const values = input.bytes.subarray(validityByteLength);
|
|
49
|
+
assertZeroPaddingBits(values, input.rowCount, "boolean bitmap");
|
|
50
|
+
for (let index = 0; index < values.byteLength; index += 1) {
|
|
51
|
+
const valueByte = values[index] ?? 0;
|
|
52
|
+
const validityByte = validity[index] ?? 0;
|
|
53
|
+
if ((valueByte & ~validityByte) !== 0) {
|
|
54
|
+
throw new Error("Boolean value is set for a null row");
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
case "number":
|
|
60
|
+
case "datetime": {
|
|
61
|
+
const expectedLength = physicalColumnByteLength(input.type, input.rowCount);
|
|
62
|
+
if (input.bytes.byteLength !== expectedLength)
|
|
63
|
+
throw new Error(`Invalid ${input.type} payload`);
|
|
64
|
+
let min = Number.POSITIVE_INFINITY;
|
|
65
|
+
let max = Number.NEGATIVE_INFINITY;
|
|
66
|
+
if (nullCount === 0 && PLATFORM_LITTLE_ENDIAN) {
|
|
67
|
+
const typedValues = alignedFloat64View(input.bytes, validityByteLength, input.rowCount);
|
|
68
|
+
for (let index = 0; index < input.rowCount; index += 1) {
|
|
69
|
+
const value = typedValues[index] ?? Number.NaN;
|
|
70
|
+
if (!Number.isFinite(value)) {
|
|
71
|
+
throw new Error(`Invalid ${input.type} physical value`);
|
|
72
|
+
}
|
|
73
|
+
if (input.type === "datetime" && (!Number.isInteger(value) || Math.abs(value) > MAX_DATETIME_MILLISECONDS)) {
|
|
74
|
+
throw new Error("Invalid datetime physical value");
|
|
75
|
+
}
|
|
76
|
+
if (value < min)
|
|
77
|
+
min = value;
|
|
78
|
+
if (value > max)
|
|
79
|
+
max = value;
|
|
67
80
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
// Every slot is a live value, so validate through a typed float view — realigned through
|
|
77
|
-
// the reusable scratch when the validity prefix leaves the payload off an 8-byte boundary.
|
|
78
|
-
const typedValues = alignedFloat64View(input.bytes, validityByteLength, input.rowCount);
|
|
79
|
-
for (let index = 0; index < input.rowCount; index += 1) {
|
|
80
|
-
const value = typedValues[index] ?? Number.NaN;
|
|
81
|
-
if (!Number.isFinite(value)) {
|
|
82
|
-
throw new Error(`Invalid ${input.type} physical value`);
|
|
83
|
-
}
|
|
84
|
-
if (input.type === "datetime" &&
|
|
85
|
-
(!Number.isInteger(value) || Math.abs(value) > MAX_DATETIME_MILLISECONDS)) {
|
|
86
|
-
throw new Error("Invalid datetime physical value");
|
|
87
|
-
}
|
|
88
|
-
if (value < min)
|
|
89
|
-
min = value;
|
|
90
|
-
if (value > max)
|
|
91
|
-
max = value;
|
|
92
|
-
}
|
|
93
|
-
metadata = zoneMapMetadata(min, max);
|
|
94
|
-
break;
|
|
95
|
-
}
|
|
96
|
-
const values = new DataView(input.bytes.buffer, input.bytes.byteOffset + validityByteLength, input.rowCount * 8);
|
|
97
|
-
for (let index = 0; index < input.rowCount; index += 1) {
|
|
98
|
-
if (!hasBit(validity, index)) {
|
|
99
|
-
assertZeroFixedWidthNullSlot(input.bytes, validityByteLength + index * 8, input.type);
|
|
100
|
-
continue;
|
|
101
|
-
}
|
|
102
|
-
const value = values.getFloat64(index * 8, true);
|
|
103
|
-
if (!Number.isFinite(value)) {
|
|
104
|
-
throw new Error(`Invalid ${input.type} physical value`);
|
|
105
|
-
}
|
|
106
|
-
if (input.type === "datetime" &&
|
|
107
|
-
(!Number.isInteger(value) || Math.abs(value) > MAX_DATETIME_MILLISECONDS)) {
|
|
108
|
-
throw new Error("Invalid datetime physical value");
|
|
109
|
-
}
|
|
110
|
-
min = Math.min(min, value);
|
|
111
|
-
max = Math.max(max, value);
|
|
112
|
-
}
|
|
113
|
-
metadata = zoneMapMetadata(min, max);
|
|
114
|
-
break;
|
|
81
|
+
metadata = zoneMapMetadata(min, max);
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
const values = new DataView(input.bytes.buffer, input.bytes.byteOffset + validityByteLength, input.rowCount * 8);
|
|
85
|
+
for (let index = 0; index < input.rowCount; index += 1) {
|
|
86
|
+
if (!hasBit(validity, index)) {
|
|
87
|
+
assertZeroFixedWidthNullSlot(input.bytes, validityByteLength + index * 8, input.type);
|
|
88
|
+
continue;
|
|
115
89
|
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
throw new Error("Invalid string offsets");
|
|
120
|
-
}
|
|
121
|
-
const offsets = new DataView(input.bytes.buffer, input.bytes.byteOffset + validityByteLength, offsetsByteLength);
|
|
122
|
-
const content = input.bytes.subarray(validityByteLength + offsetsByteLength);
|
|
123
|
-
let previous = offsets.getUint32(0, true);
|
|
124
|
-
if (previous !== 0)
|
|
125
|
-
throw new Error("Invalid first string offset");
|
|
126
|
-
for (let index = 0; index < input.rowCount; index += 1) {
|
|
127
|
-
const current = offsets.getUint32((index + 1) * 4, true);
|
|
128
|
-
if (current < previous || current > content.byteLength) {
|
|
129
|
-
throw new Error("Invalid string offset");
|
|
130
|
-
}
|
|
131
|
-
if (hasBit(validity, index)) {
|
|
132
|
-
assertValidUtf8(content, previous, current);
|
|
133
|
-
}
|
|
134
|
-
else if (current !== previous) {
|
|
135
|
-
throw new Error("Null strings must have empty content");
|
|
136
|
-
}
|
|
137
|
-
previous = current;
|
|
138
|
-
}
|
|
139
|
-
if (previous !== content.byteLength)
|
|
140
|
-
throw new Error("Unreferenced string bytes");
|
|
141
|
-
physicalColumnByteLength(input.type, input.rowCount, content.byteLength);
|
|
142
|
-
break;
|
|
90
|
+
const value = values.getFloat64(index * 8, true);
|
|
91
|
+
if (!Number.isFinite(value)) {
|
|
92
|
+
throw new Error(`Invalid ${input.type} physical value`);
|
|
143
93
|
}
|
|
94
|
+
if (input.type === "datetime" && (!Number.isInteger(value) || Math.abs(value) > MAX_DATETIME_MILLISECONDS)) {
|
|
95
|
+
throw new Error("Invalid datetime physical value");
|
|
96
|
+
}
|
|
97
|
+
min = Math.min(min, value);
|
|
98
|
+
max = Math.max(max, value);
|
|
99
|
+
}
|
|
100
|
+
metadata = zoneMapMetadata(min, max);
|
|
101
|
+
break;
|
|
144
102
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
103
|
+
case "string": {
|
|
104
|
+
const offsetsByteLength = checkedMultiply(checkedAdd(input.rowCount, 1, "string offsets"), 4, "string offsets");
|
|
105
|
+
if (input.bytes.byteLength < validityByteLength + offsetsByteLength) {
|
|
106
|
+
throw new Error("Invalid string offsets");
|
|
107
|
+
}
|
|
108
|
+
const offsets = new DataView(input.bytes.buffer, input.bytes.byteOffset + validityByteLength, offsetsByteLength);
|
|
109
|
+
const content = input.bytes.subarray(validityByteLength + offsetsByteLength);
|
|
110
|
+
let previous = offsets.getUint32(0, true);
|
|
111
|
+
if (previous !== 0)
|
|
112
|
+
throw new Error("Invalid first string offset");
|
|
113
|
+
for (let index = 0; index < input.rowCount; index += 1) {
|
|
114
|
+
const current = offsets.getUint32((index + 1) * 4, true);
|
|
115
|
+
if (current < previous || current > content.byteLength) {
|
|
116
|
+
throw new Error("Invalid string offset");
|
|
117
|
+
}
|
|
118
|
+
if (hasBit(validity, index)) {
|
|
119
|
+
assertValidUtf8(content, previous, current);
|
|
120
|
+
} else if (current !== previous) {
|
|
121
|
+
throw new Error("Null strings must have empty content");
|
|
122
|
+
}
|
|
123
|
+
previous = current;
|
|
124
|
+
}
|
|
125
|
+
if (previous !== content.byteLength)
|
|
126
|
+
throw new Error("Unreferenced string bytes");
|
|
127
|
+
physicalColumnByteLength(input.type, input.rowCount, content.byteLength);
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return {
|
|
132
|
+
type: input.type,
|
|
133
|
+
rowCount: input.rowCount,
|
|
134
|
+
nullCount,
|
|
135
|
+
bytes: input.bytes,
|
|
136
|
+
metadata
|
|
137
|
+
};
|
|
152
138
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
* No output byte buffer is allocated.
|
|
156
|
-
*/
|
|
157
|
-
export function measurePhysicalColumnRanges(type, ranges) {
|
|
158
|
-
return preparePhysicalColumn(type, ranges).measurement;
|
|
139
|
+
function measurePhysicalColumnRanges(type, ranges) {
|
|
140
|
+
return preparePhysicalColumn(type, ranges).measurement;
|
|
159
141
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
142
|
+
function buildPhysicalColumnFromRanges(type, ranges) {
|
|
143
|
+
const prepared = preparePhysicalColumn(type, ranges);
|
|
144
|
+
const { measurement } = prepared;
|
|
145
|
+
const output = new Uint8Array(measurement.encodedByteLength);
|
|
146
|
+
const outputValidity = output.subarray(0, measurement.validityByteLength);
|
|
147
|
+
let outputRow = 0;
|
|
148
|
+
for (const range of prepared.ranges) {
|
|
149
|
+
const sourceValidityLength = physicalBitmapByteLength(range.column.rowCount);
|
|
150
|
+
const sourceValidity = range.column.bytes.subarray(0, sourceValidityLength);
|
|
151
|
+
for (let sourceRow = range.start; sourceRow < range.end; sourceRow += 1) {
|
|
152
|
+
if (hasBit(sourceValidity, sourceRow))
|
|
153
|
+
setBit(outputValidity, outputRow);
|
|
154
|
+
outputRow += 1;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
switch (type) {
|
|
158
|
+
case "boolean": {
|
|
159
|
+
const outputValues = output.subarray(measurement.validityByteLength);
|
|
160
|
+
outputRow = 0;
|
|
161
|
+
for (const range of prepared.ranges) {
|
|
162
|
+
const sourceBitmapLength = physicalBitmapByteLength(range.column.rowCount);
|
|
163
|
+
const sourceValues = range.column.bytes.subarray(sourceBitmapLength);
|
|
170
164
|
for (let sourceRow = range.start; sourceRow < range.end; sourceRow += 1) {
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
165
|
+
if (hasBit(sourceValues, sourceRow))
|
|
166
|
+
setBit(outputValues, outputRow);
|
|
167
|
+
outputRow += 1;
|
|
174
168
|
}
|
|
169
|
+
}
|
|
170
|
+
break;
|
|
175
171
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
let contentByteLength = 0;
|
|
209
|
-
outputRow = 0;
|
|
210
|
-
outputOffsets.setUint32(0, 0, true);
|
|
211
|
-
for (const range of prepared.ranges) {
|
|
212
|
-
const sourceValidityLength = physicalBitmapByteLength(range.column.rowCount);
|
|
213
|
-
const sourceOffsetsByteLength = (range.column.rowCount + 1) * 4;
|
|
214
|
-
const sourceOffsets = new DataView(range.column.bytes.buffer, range.column.bytes.byteOffset + sourceValidityLength, sourceOffsetsByteLength);
|
|
215
|
-
const sourceContent = range.column.bytes.subarray(sourceValidityLength + sourceOffsetsByteLength);
|
|
216
|
-
const sourceContentStart = sourceOffsets.getUint32(range.start * 4, true);
|
|
217
|
-
const sourceContentEnd = sourceOffsets.getUint32(range.end * 4, true);
|
|
218
|
-
output.set(sourceContent.subarray(sourceContentStart, sourceContentEnd), outputContentOffset + contentByteLength);
|
|
219
|
-
for (let sourceRow = range.start; sourceRow < range.end; sourceRow += 1) {
|
|
220
|
-
const sourceEnd = sourceOffsets.getUint32((sourceRow + 1) * 4, true);
|
|
221
|
-
outputRow += 1;
|
|
222
|
-
outputOffsets.setUint32(outputRow * 4, contentByteLength + sourceEnd - sourceContentStart, true);
|
|
223
|
-
}
|
|
224
|
-
contentByteLength += sourceContentEnd - sourceContentStart;
|
|
225
|
-
}
|
|
226
|
-
break;
|
|
172
|
+
case "number":
|
|
173
|
+
case "datetime": {
|
|
174
|
+
let outputValueOffset = measurement.validityByteLength;
|
|
175
|
+
for (const range of prepared.ranges) {
|
|
176
|
+
const sourceValidityLength = physicalBitmapByteLength(range.column.rowCount);
|
|
177
|
+
const start = sourceValidityLength + range.start * 8;
|
|
178
|
+
const end = sourceValidityLength + range.end * 8;
|
|
179
|
+
const sourceValues = range.column.bytes.subarray(start, end);
|
|
180
|
+
output.set(sourceValues, outputValueOffset);
|
|
181
|
+
outputValueOffset += sourceValues.byteLength;
|
|
182
|
+
}
|
|
183
|
+
break;
|
|
184
|
+
}
|
|
185
|
+
case "string": {
|
|
186
|
+
const outputOffsetsByteLength = (measurement.rowCount + 1) * 4;
|
|
187
|
+
const outputOffsets = new DataView(output.buffer, output.byteOffset + measurement.validityByteLength, outputOffsetsByteLength);
|
|
188
|
+
const outputContentOffset = measurement.validityByteLength + outputOffsetsByteLength;
|
|
189
|
+
let contentByteLength = 0;
|
|
190
|
+
outputRow = 0;
|
|
191
|
+
outputOffsets.setUint32(0, 0, true);
|
|
192
|
+
for (const range of prepared.ranges) {
|
|
193
|
+
const sourceValidityLength = physicalBitmapByteLength(range.column.rowCount);
|
|
194
|
+
const sourceOffsetsByteLength = (range.column.rowCount + 1) * 4;
|
|
195
|
+
const sourceOffsets = new DataView(range.column.bytes.buffer, range.column.bytes.byteOffset + sourceValidityLength, sourceOffsetsByteLength);
|
|
196
|
+
const sourceContent = range.column.bytes.subarray(sourceValidityLength + sourceOffsetsByteLength);
|
|
197
|
+
const sourceContentStart = sourceOffsets.getUint32(range.start * 4, true);
|
|
198
|
+
const sourceContentEnd = sourceOffsets.getUint32(range.end * 4, true);
|
|
199
|
+
output.set(sourceContent.subarray(sourceContentStart, sourceContentEnd), outputContentOffset + contentByteLength);
|
|
200
|
+
for (let sourceRow = range.start; sourceRow < range.end; sourceRow += 1) {
|
|
201
|
+
const sourceEnd = sourceOffsets.getUint32((sourceRow + 1) * 4, true);
|
|
202
|
+
outputRow += 1;
|
|
203
|
+
outputOffsets.setUint32(outputRow * 4, contentByteLength + sourceEnd - sourceContentStart, true);
|
|
227
204
|
}
|
|
205
|
+
contentByteLength += sourceContentEnd - sourceContentStart;
|
|
206
|
+
}
|
|
207
|
+
break;
|
|
228
208
|
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
type,
|
|
212
|
+
rowCount: measurement.rowCount,
|
|
213
|
+
nullCount: measurement.nullCount,
|
|
214
|
+
bytes: output,
|
|
215
|
+
metadata: measurement.metadata
|
|
216
|
+
};
|
|
236
217
|
}
|
|
237
|
-
|
|
238
|
-
|
|
218
|
+
function slicePhysicalColumn(column, start, end) {
|
|
219
|
+
return buildPhysicalColumnFromRanges(column.type, [{ column, start, end }]);
|
|
239
220
|
}
|
|
240
|
-
|
|
241
|
-
|
|
221
|
+
function concatenatePhysicalColumns(type, columns) {
|
|
222
|
+
return buildPhysicalColumnFromRanges(type, columns.map((column) => ({ column, start: 0, end: column.rowCount })));
|
|
242
223
|
}
|
|
243
224
|
function preparePhysicalColumn(type, ranges) {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
}
|
|
271
|
-
const sourceValues = type === "number" || type === "datetime"
|
|
272
|
-
? new DataView(column.bytes.buffer, column.bytes.byteOffset + sourceValidityLength, column.rowCount * 8)
|
|
273
|
-
: undefined;
|
|
274
|
-
for (let sourceRow = range.start; sourceRow < range.end; sourceRow += 1) {
|
|
275
|
-
if (!hasBit(sourceValidity, sourceRow)) {
|
|
276
|
-
nullCount += 1;
|
|
277
|
-
}
|
|
278
|
-
else if (sourceValues !== undefined) {
|
|
279
|
-
const value = sourceValues.getFloat64(sourceRow * 8, true);
|
|
280
|
-
min = Math.min(min, value);
|
|
281
|
-
max = Math.max(max, value);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
preparedRanges.push({ column, start: range.start, end: range.end });
|
|
225
|
+
const preparedRanges = [];
|
|
226
|
+
let rowCount = 0;
|
|
227
|
+
let nullCount = 0;
|
|
228
|
+
let stringContentByteLength = 0;
|
|
229
|
+
let min = Number.POSITIVE_INFINITY;
|
|
230
|
+
let max = Number.NEGATIVE_INFINITY;
|
|
231
|
+
for (const range of ranges) {
|
|
232
|
+
if (range.column.type !== type) {
|
|
233
|
+
throw new TypeError(`Cannot append ${range.column.type} physical rows to a ${type} column`);
|
|
234
|
+
}
|
|
235
|
+
const column = validatePhysicalColumn(range.column);
|
|
236
|
+
assertRangeBoundary(range.start, "range start");
|
|
237
|
+
assertRangeBoundary(range.end, "range end");
|
|
238
|
+
if (range.start > range.end || range.end > column.rowCount) {
|
|
239
|
+
throw new RangeError("Physical row range is out of bounds");
|
|
240
|
+
}
|
|
241
|
+
const rangeRowCount = range.end - range.start;
|
|
242
|
+
rowCount = checkedAdd(rowCount, rangeRowCount, "row count");
|
|
243
|
+
assertBlockRowCount(rowCount);
|
|
244
|
+
const sourceValidityLength = physicalBitmapByteLength(column.rowCount);
|
|
245
|
+
const sourceValidity = column.bytes.subarray(0, sourceValidityLength);
|
|
246
|
+
if (type === "string") {
|
|
247
|
+
const sourceOffsets = new DataView(column.bytes.buffer, column.bytes.byteOffset + sourceValidityLength, (column.rowCount + 1) * 4);
|
|
248
|
+
const contentStart = sourceOffsets.getUint32(range.start * 4, true);
|
|
249
|
+
const contentEnd = sourceOffsets.getUint32(range.end * 4, true);
|
|
250
|
+
stringContentByteLength = checkedAdd(stringContentByteLength, contentEnd - contentStart, "string content byte length");
|
|
285
251
|
}
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
252
|
+
const sourceValues = type === "number" || type === "datetime" ? new DataView(column.bytes.buffer, column.bytes.byteOffset + sourceValidityLength, column.rowCount * 8) : void 0;
|
|
253
|
+
for (let sourceRow = range.start; sourceRow < range.end; sourceRow += 1) {
|
|
254
|
+
if (!hasBit(sourceValidity, sourceRow)) {
|
|
255
|
+
nullCount += 1;
|
|
256
|
+
} else if (sourceValues !== void 0) {
|
|
257
|
+
const value = sourceValues.getFloat64(sourceRow * 8, true);
|
|
258
|
+
min = Math.min(min, value);
|
|
259
|
+
max = Math.max(max, value);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
preparedRanges.push({ column, start: range.start, end: range.end });
|
|
263
|
+
}
|
|
264
|
+
const validityByteLength = physicalBitmapByteLength(rowCount);
|
|
265
|
+
const encodedByteLength = physicalColumnByteLength(type, rowCount, stringContentByteLength);
|
|
266
|
+
const valueByteLength = encodedByteLength - validityByteLength;
|
|
267
|
+
return {
|
|
268
|
+
ranges: preparedRanges,
|
|
269
|
+
measurement: {
|
|
270
|
+
type,
|
|
271
|
+
rowCount,
|
|
272
|
+
nullCount,
|
|
273
|
+
validityByteLength,
|
|
274
|
+
valueByteLength,
|
|
275
|
+
stringContentByteLength,
|
|
276
|
+
encodedByteLength,
|
|
277
|
+
metadata: type === "number" || type === "datetime" ? zoneMapMetadata(min, max) : {}
|
|
278
|
+
}
|
|
279
|
+
};
|
|
302
280
|
}
|
|
303
281
|
function zoneMapMetadata(min, max) {
|
|
304
|
-
|
|
282
|
+
return min === Number.POSITIVE_INFINITY ? {} : { zoneMap: { min, max } };
|
|
305
283
|
}
|
|
306
|
-
const PLATFORM_LITTLE_ENDIAN = new Uint8Array(Float64Array.of(1).buffer)[7] ===
|
|
284
|
+
const PLATFORM_LITTLE_ENDIAN = new Uint8Array(Float64Array.of(1).buffer)[7] === 63;
|
|
307
285
|
let float64AlignmentScratch = new ArrayBuffer(0);
|
|
308
|
-
/**
|
|
309
|
-
* Returns a Float64Array over `rowCount` values starting at `byteOffset` within `bytes`,
|
|
310
|
-
* realigning through a reusable scratch copy when the source is not 8-byte aligned.
|
|
311
|
-
*/
|
|
312
286
|
function alignedFloat64View(bytes, byteOffset, rowCount) {
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
287
|
+
const start = bytes.byteOffset + byteOffset;
|
|
288
|
+
if (start % Float64Array.BYTES_PER_ELEMENT === 0) {
|
|
289
|
+
return new Float64Array(bytes.buffer, start, rowCount);
|
|
290
|
+
}
|
|
291
|
+
const byteLength = rowCount * Float64Array.BYTES_PER_ELEMENT;
|
|
292
|
+
if (float64AlignmentScratch.byteLength < byteLength) {
|
|
293
|
+
float64AlignmentScratch = new ArrayBuffer(byteLength);
|
|
294
|
+
}
|
|
295
|
+
new Uint8Array(float64AlignmentScratch, 0, byteLength).set(new Uint8Array(bytes.buffer, start, byteLength));
|
|
296
|
+
return new Float64Array(float64AlignmentScratch, 0, rowCount);
|
|
323
297
|
}
|
|
324
298
|
const POPCOUNT = new Uint8Array(256);
|
|
325
299
|
for (let value = 0; value < 256; value += 1) {
|
|
326
|
-
|
|
300
|
+
POPCOUNT[value] = (POPCOUNT[value >> 1] ?? 0) + (value & 1);
|
|
327
301
|
}
|
|
328
302
|
function countSetBits(bytes, rowCount) {
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
return count;
|
|
303
|
+
const wholeBytes = rowCount >>> 3;
|
|
304
|
+
let count = 0;
|
|
305
|
+
for (let index = 0; index < wholeBytes; index += 1) {
|
|
306
|
+
count += POPCOUNT[bytes[index] ?? 0] ?? 0;
|
|
307
|
+
}
|
|
308
|
+
for (let index = wholeBytes * 8; index < rowCount; index += 1) {
|
|
309
|
+
if (hasBit(bytes, index))
|
|
310
|
+
count += 1;
|
|
311
|
+
}
|
|
312
|
+
return count;
|
|
340
313
|
}
|
|
341
314
|
function setBit(bitmap, index) {
|
|
342
|
-
|
|
343
|
-
|
|
315
|
+
const byteOffset = index >>> 3;
|
|
316
|
+
bitmap[byteOffset] = (bitmap[byteOffset] ?? 0) | 1 << (index & 7);
|
|
344
317
|
}
|
|
345
318
|
function hasBit(bitmap, index) {
|
|
346
|
-
|
|
319
|
+
return ((bitmap[index >>> 3] ?? 0) & 1 << (index & 7)) !== 0;
|
|
347
320
|
}
|
|
348
321
|
function assertZeroPaddingBits(bitmap, rowCount, name) {
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
322
|
+
const usedBits = rowCount & 7;
|
|
323
|
+
if (usedBits === 0 || bitmap.byteLength === 0)
|
|
324
|
+
return;
|
|
325
|
+
const finalByte = bitmap[bitmap.byteLength - 1] ?? 0;
|
|
326
|
+
const usedMask = (1 << usedBits) - 1;
|
|
327
|
+
if ((finalByte & ~usedMask) !== 0)
|
|
328
|
+
throw new Error(`Non-zero padding in ${name}`);
|
|
356
329
|
}
|
|
357
330
|
function assertZeroFixedWidthNullSlot(bytes, offset, type) {
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
331
|
+
for (let index = offset; index < offset + 8; index += 1) {
|
|
332
|
+
if (bytes[index] !== 0)
|
|
333
|
+
throw new Error(`${type} value bytes are set for a null row`);
|
|
334
|
+
}
|
|
362
335
|
}
|
|
363
336
|
function assertValidUtf8(bytes, start, end) {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
throw new Error("Invalid UTF-8 string payload");
|
|
380
|
-
}
|
|
381
|
-
assertContinuation(bytes, index + 2, end);
|
|
382
|
-
index += 3;
|
|
383
|
-
continue;
|
|
384
|
-
}
|
|
385
|
-
if (first >= 0xf0 && first <= 0xf4) {
|
|
386
|
-
const second = continuation(bytes, index + 1, end);
|
|
387
|
-
if ((first === 0xf0 && second < 0x90) || (first === 0xf4 && second > 0x8f)) {
|
|
388
|
-
throw new Error("Invalid UTF-8 string payload");
|
|
389
|
-
}
|
|
390
|
-
assertContinuation(bytes, index + 2, end);
|
|
391
|
-
assertContinuation(bytes, index + 3, end);
|
|
392
|
-
index += 4;
|
|
393
|
-
continue;
|
|
394
|
-
}
|
|
337
|
+
let index = start;
|
|
338
|
+
while (index < end) {
|
|
339
|
+
const first = bytes[index] ?? 0;
|
|
340
|
+
if (first <= 127) {
|
|
341
|
+
index += 1;
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
if (first >= 194 && first <= 223) {
|
|
345
|
+
assertContinuation(bytes, index + 1, end);
|
|
346
|
+
index += 2;
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (first >= 224 && first <= 239) {
|
|
350
|
+
const second = continuation(bytes, index + 1, end);
|
|
351
|
+
if (first === 224 && second < 160 || first === 237 && second > 159) {
|
|
395
352
|
throw new Error("Invalid UTF-8 string payload");
|
|
353
|
+
}
|
|
354
|
+
assertContinuation(bytes, index + 2, end);
|
|
355
|
+
index += 3;
|
|
356
|
+
continue;
|
|
396
357
|
}
|
|
358
|
+
if (first >= 240 && first <= 244) {
|
|
359
|
+
const second = continuation(bytes, index + 1, end);
|
|
360
|
+
if (first === 240 && second < 144 || first === 244 && second > 143) {
|
|
361
|
+
throw new Error("Invalid UTF-8 string payload");
|
|
362
|
+
}
|
|
363
|
+
assertContinuation(bytes, index + 2, end);
|
|
364
|
+
assertContinuation(bytes, index + 3, end);
|
|
365
|
+
index += 4;
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
throw new Error("Invalid UTF-8 string payload");
|
|
369
|
+
}
|
|
397
370
|
}
|
|
398
371
|
function continuation(bytes, index, end) {
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
372
|
+
if (index >= end)
|
|
373
|
+
throw new Error("Invalid UTF-8 string payload");
|
|
374
|
+
const byte = bytes[index] ?? 0;
|
|
375
|
+
if (byte < 128 || byte > 191)
|
|
376
|
+
throw new Error("Invalid UTF-8 string payload");
|
|
377
|
+
return byte;
|
|
405
378
|
}
|
|
406
379
|
function assertContinuation(bytes, index, end) {
|
|
407
|
-
|
|
380
|
+
continuation(bytes, index, end);
|
|
408
381
|
}
|
|
409
382
|
function assertRowCount(rowCount) {
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
383
|
+
assertNonnegativeSafeInteger(rowCount, "row count");
|
|
384
|
+
if (rowCount > MAX_BLOCK_ROW_COUNT) {
|
|
385
|
+
throw new RangeError("Physical column exceeds maximum row count");
|
|
386
|
+
}
|
|
414
387
|
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
assertRowCount(rowCount);
|
|
388
|
+
function assertBlockRowCount(rowCount) {
|
|
389
|
+
assertRowCount(rowCount);
|
|
418
390
|
}
|
|
419
391
|
function assertLogicalType(type) {
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
392
|
+
if (type !== "boolean" && type !== "number" && type !== "string" && type !== "datetime") {
|
|
393
|
+
throw new TypeError(`Unknown logical type: ${String(type)}`);
|
|
394
|
+
}
|
|
423
395
|
}
|
|
424
396
|
function assertRangeBoundary(value, name) {
|
|
425
|
-
|
|
397
|
+
assertNonnegativeSafeInteger(value, name);
|
|
426
398
|
}
|
|
427
399
|
function assertNonnegativeSafeInteger(value, name) {
|
|
428
|
-
|
|
429
|
-
|
|
400
|
+
if (!Number.isSafeInteger(value) || value < 0)
|
|
401
|
+
throw new RangeError(`Invalid ${name}`);
|
|
430
402
|
}
|
|
431
403
|
function assertPhysicalByteLength(byteLength) {
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
404
|
+
assertNonnegativeSafeInteger(byteLength, "physical column byte length");
|
|
405
|
+
if (byteLength > MAX_PHYSICAL_COLUMN_BYTE_LENGTH) {
|
|
406
|
+
throw new RangeError("Physical column exceeds maximum byte length");
|
|
407
|
+
}
|
|
436
408
|
}
|
|
437
409
|
function checkedAdd(left, right, name) {
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
410
|
+
if (left > Number.MAX_SAFE_INTEGER - right)
|
|
411
|
+
throw new RangeError(`Invalid ${name} length`);
|
|
412
|
+
return left + right;
|
|
441
413
|
}
|
|
442
414
|
function checkedMultiply(left, right, name) {
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
415
|
+
if (left !== 0 && right > Math.floor(Number.MAX_SAFE_INTEGER / left)) {
|
|
416
|
+
throw new RangeError(`Invalid ${name} length`);
|
|
417
|
+
}
|
|
418
|
+
return left * right;
|
|
447
419
|
}
|
|
420
|
+
export {
|
|
421
|
+
MAX_BLOCK_ROW_COUNT,
|
|
422
|
+
MAX_PHYSICAL_COLUMN_BYTE_LENGTH,
|
|
423
|
+
assertBlockRowCount,
|
|
424
|
+
buildPhysicalColumnFromRanges,
|
|
425
|
+
concatenatePhysicalColumns,
|
|
426
|
+
measurePhysicalColumnRanges,
|
|
427
|
+
physicalBitmapByteLength,
|
|
428
|
+
physicalColumnByteLength,
|
|
429
|
+
slicePhysicalColumn,
|
|
430
|
+
validatePhysicalColumn
|
|
431
|
+
};
|