@konstit/dne 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -351
- package/data/dne.bin +0 -0
- package/dist/bairro-runs.d.ts +20 -0
- package/dist/bairro-runs.js +202 -0
- package/dist/bairro-runs.js.map +1 -0
- package/dist/binary-db-format.d.ts +10 -2
- package/dist/binary-db-format.js +9 -1
- package/dist/binary-db-format.js.map +1 -1
- package/dist/binary-db-integrity.d.ts +4 -0
- package/dist/binary-db-integrity.js +20 -0
- package/dist/binary-db-integrity.js.map +1 -0
- package/dist/binary-db-reader.d.ts +29 -17
- package/dist/binary-db-reader.js +191 -335
- package/dist/binary-db-reader.js.map +1 -1
- package/dist/binary-db-validator.d.ts +13 -0
- package/dist/binary-db-validator.js +712 -0
- package/dist/binary-db-validator.js.map +1 -0
- package/dist/fsst.d.ts +18 -0
- package/dist/fsst.js +117 -0
- package/dist/fsst.js.map +1 -0
- package/dist/index.js +44 -44
- package/dist/index.js.map +16 -9
- package/dist/locality-runs.d.ts +18 -0
- package/dist/locality-runs.js +108 -0
- package/dist/locality-runs.js.map +1 -0
- package/dist/packed-bits.d.ts +6 -0
- package/dist/packed-bits.js +41 -0
- package/dist/packed-bits.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { DneBinaryDatabaseFormatError } from "./binary-db-errors.js";
|
|
2
|
+
import { createPopcountTable, readPackedInteger, writePackedInteger, } from "./binary-db-format.js";
|
|
3
|
+
const HEADER_BYTES = 8;
|
|
4
|
+
const SAMPLE_SHIFT = 3;
|
|
5
|
+
const SAMPLE_STEP = 1 << SAMPLE_SHIFT;
|
|
6
|
+
const popcount = createPopcountTable();
|
|
7
|
+
/** Encodes consecutive equal neighborhood indexes; Elias–Fano stores each run's starting row. */
|
|
8
|
+
export function encodeBairroRuns(dense, rowCount, idWidth) {
|
|
9
|
+
const view = new DataView(dense.buffer, dense.byteOffset, dense.byteLength);
|
|
10
|
+
const starts = [];
|
|
11
|
+
const ids = [];
|
|
12
|
+
let previous = -1;
|
|
13
|
+
for (let row = 0; row < rowCount; row++) {
|
|
14
|
+
const id = readPackedInteger(view, row * idWidth, idWidth);
|
|
15
|
+
if (id !== previous) {
|
|
16
|
+
starts.push(row);
|
|
17
|
+
ids.push(id);
|
|
18
|
+
previous = id;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const runCount = starts.length;
|
|
22
|
+
const lowBits = Math.floor(Math.log2(rowCount / runCount));
|
|
23
|
+
const lowBase = 2 ** lowBits;
|
|
24
|
+
const zeroCount = Math.ceil(rowCount / lowBase);
|
|
25
|
+
const highBitCount = zeroCount + runCount;
|
|
26
|
+
const lowBytes = Math.ceil(runCount * lowBits / 8);
|
|
27
|
+
const highBytes = Math.ceil(highBitCount / 8);
|
|
28
|
+
const sampleCount = Math.ceil(zeroCount / SAMPLE_STEP);
|
|
29
|
+
const highOffset = HEADER_BYTES + lowBytes;
|
|
30
|
+
const samplesOffset = highOffset + highBytes;
|
|
31
|
+
const idsOffset = samplesOffset + sampleCount * 4;
|
|
32
|
+
const bytes = new Uint8Array(idsOffset + runCount * idWidth);
|
|
33
|
+
const data = new DataView(bytes.buffer);
|
|
34
|
+
data.setUint32(0, runCount, true);
|
|
35
|
+
data.setUint8(4, lowBits);
|
|
36
|
+
data.setUint8(5, SAMPLE_SHIFT);
|
|
37
|
+
data.setUint8(6, idWidth);
|
|
38
|
+
for (let index = 0; index < runCount; index++) {
|
|
39
|
+
const start = starts[index] ?? 0;
|
|
40
|
+
writeBits(bytes, HEADER_BYTES, index * lowBits, lowBits, start % lowBase);
|
|
41
|
+
const highBit = Math.floor(start / lowBase) + index;
|
|
42
|
+
const highByte = highOffset + (highBit >>> 3);
|
|
43
|
+
bytes[highByte] = (bytes[highByte] ?? 0) | (1 << (highBit & 7));
|
|
44
|
+
writePackedInteger(bytes, idsOffset + index * idWidth, ids[index] ?? 0, idWidth);
|
|
45
|
+
}
|
|
46
|
+
let zeroIndex = 0;
|
|
47
|
+
for (let bit = 0; bit < highBitCount; bit++) {
|
|
48
|
+
if (((bytes[highOffset + (bit >>> 3)] ?? 0) & (1 << (bit & 7))) === 0) {
|
|
49
|
+
if (zeroIndex % SAMPLE_STEP === 0) {
|
|
50
|
+
data.setUint32(samplesOffset + (zeroIndex >>> SAMPLE_SHIFT) * 4, bit, true);
|
|
51
|
+
}
|
|
52
|
+
zeroIndex++;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return bytes;
|
|
56
|
+
}
|
|
57
|
+
/** Reads locations after the file has passed integrity verification. */
|
|
58
|
+
export function readBairroRunsLayout(data, region, rowCount, idWidth) {
|
|
59
|
+
const runCount = data.getUint32(region.offset, true);
|
|
60
|
+
const lowBits = data.getUint8(region.offset + 4);
|
|
61
|
+
const zeroCount = Math.ceil(rowCount / 2 ** lowBits);
|
|
62
|
+
const highBitCount = zeroCount + runCount;
|
|
63
|
+
const lowOffset = region.offset + HEADER_BYTES;
|
|
64
|
+
const highOffset = lowOffset + Math.ceil(runCount * lowBits / 8);
|
|
65
|
+
const samplesOffset = highOffset + Math.ceil(highBitCount / 8);
|
|
66
|
+
const idsOffset = samplesOffset + Math.ceil(zeroCount / SAMPLE_STEP) * 4;
|
|
67
|
+
return { highBitCount, highOffset, idWidth, idsOffset, lowBits, lowOffset, runCount, samplesOffset, zeroCount };
|
|
68
|
+
}
|
|
69
|
+
/** Validates the complete encoded run index during generation. */
|
|
70
|
+
export function parseBairroRuns(data, region, rowCount, idWidth, bairroCount) {
|
|
71
|
+
if (region.length < HEADER_BYTES) {
|
|
72
|
+
throw new DneBinaryDatabaseFormatError('Binary neighborhood runs are truncated');
|
|
73
|
+
}
|
|
74
|
+
const runCount = data.getUint32(region.offset, true);
|
|
75
|
+
const lowBits = data.getUint8(region.offset + 4);
|
|
76
|
+
if (runCount < 1 || runCount > rowCount || lowBits > 31
|
|
77
|
+
|| data.getUint8(region.offset + 5) !== SAMPLE_SHIFT
|
|
78
|
+
|| data.getUint8(region.offset + 6) !== idWidth
|
|
79
|
+
|| data.getUint8(region.offset + 7) !== 0) {
|
|
80
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run header');
|
|
81
|
+
}
|
|
82
|
+
const lowBase = 2 ** lowBits;
|
|
83
|
+
const zeroCount = Math.ceil(rowCount / lowBase);
|
|
84
|
+
const highBitCount = zeroCount + runCount;
|
|
85
|
+
const lowBytes = Math.ceil(runCount * lowBits / 8);
|
|
86
|
+
const highBytes = Math.ceil(highBitCount / 8);
|
|
87
|
+
const sampleCount = Math.ceil(zeroCount / SAMPLE_STEP);
|
|
88
|
+
const lowOffset = region.offset + HEADER_BYTES;
|
|
89
|
+
const highOffset = lowOffset + lowBytes;
|
|
90
|
+
const samplesOffset = highOffset + highBytes;
|
|
91
|
+
const idsOffset = samplesOffset + sampleCount * 4;
|
|
92
|
+
if (idsOffset + runCount * idWidth !== region.offset + region.length) {
|
|
93
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run length');
|
|
94
|
+
}
|
|
95
|
+
let zeros = 0;
|
|
96
|
+
let runs = 0;
|
|
97
|
+
let previousStart = -1;
|
|
98
|
+
let previousId = -1;
|
|
99
|
+
for (let bit = 0; bit < highBitCount; bit++) {
|
|
100
|
+
const one = (data.getUint8(highOffset + (bit >>> 3)) & (1 << (bit & 7))) !== 0;
|
|
101
|
+
if (!one) {
|
|
102
|
+
if (zeros >= zeroCount) {
|
|
103
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run bitmap');
|
|
104
|
+
}
|
|
105
|
+
if (zeros % SAMPLE_STEP === 0 && data.getUint32(samplesOffset + (zeros >>> SAMPLE_SHIFT) * 4, true) !== bit) {
|
|
106
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run samples');
|
|
107
|
+
}
|
|
108
|
+
zeros++;
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
if (runs >= runCount) {
|
|
112
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run bitmap');
|
|
113
|
+
}
|
|
114
|
+
const low = readBits(data, lowOffset, runs * lowBits, lowBits);
|
|
115
|
+
const start = zeros * lowBase + low;
|
|
116
|
+
const id = readPackedInteger(data, idsOffset + runs * idWidth, idWidth);
|
|
117
|
+
if ((runs === 0 && start !== 0) || start <= previousStart || start >= rowCount || id > bairroCount || id === previousId) {
|
|
118
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run');
|
|
119
|
+
}
|
|
120
|
+
previousStart = start;
|
|
121
|
+
previousId = id;
|
|
122
|
+
runs++;
|
|
123
|
+
}
|
|
124
|
+
if (highBitCount % 8 !== 0) {
|
|
125
|
+
const paddingMask = 0xff << (highBitCount & 7);
|
|
126
|
+
if ((data.getUint8(highOffset + (highBitCount >>> 3)) & paddingMask) !== 0) {
|
|
127
|
+
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run padding');
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return { highBitCount, highOffset, idWidth, idsOffset, lowBits, lowOffset, runCount, samplesOffset, zeroCount };
|
|
131
|
+
}
|
|
132
|
+
/** Finds the run containing a row using the Elias–Fano high bucket and its packed low values. */
|
|
133
|
+
export function readBairroRunIndex(data, runs, row) {
|
|
134
|
+
const lowBase = 2 ** runs.lowBits;
|
|
135
|
+
const high = Math.floor(row / lowBase);
|
|
136
|
+
const end = selectZero(data, runs, high) - high;
|
|
137
|
+
const start = high === 0 ? 0 : selectZero(data, runs, high - 1) - (high - 1);
|
|
138
|
+
const wantedLow = row % lowBase;
|
|
139
|
+
let low = start;
|
|
140
|
+
let upper = end;
|
|
141
|
+
while (low < upper) {
|
|
142
|
+
const middle = low + ((upper - low) >>> 1);
|
|
143
|
+
const value = readBits(data, runs.lowOffset, middle * runs.lowBits, runs.lowBits);
|
|
144
|
+
if (value <= wantedLow) {
|
|
145
|
+
low = middle + 1;
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
upper = middle;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const runIndex = low - 1;
|
|
152
|
+
return readPackedInteger(data, runs.idsOffset + runIndex * runs.idWidth, runs.idWidth);
|
|
153
|
+
}
|
|
154
|
+
function selectZero(data, runs, zeroIndex) {
|
|
155
|
+
const sampleIndex = zeroIndex >>> SAMPLE_SHIFT;
|
|
156
|
+
let bit = data.getUint32(runs.samplesOffset + sampleIndex * 4, true);
|
|
157
|
+
let remaining = zeroIndex - sampleIndex * SAMPLE_STEP;
|
|
158
|
+
if (remaining === 0) {
|
|
159
|
+
return bit;
|
|
160
|
+
}
|
|
161
|
+
bit++;
|
|
162
|
+
while (remaining > 0) {
|
|
163
|
+
const withinByte = bit & 7;
|
|
164
|
+
const available = 8 - withinByte;
|
|
165
|
+
const mask = (1 << available) - 1;
|
|
166
|
+
const zeroBits = (~(data.getUint8(runs.highOffset + (bit >>> 3)) >>> withinByte)) & mask;
|
|
167
|
+
const count = popcount[zeroBits] ?? 0;
|
|
168
|
+
if (count >= remaining) {
|
|
169
|
+
for (let offset = 0; remaining > 0; offset++) {
|
|
170
|
+
if (zeroBits & (1 << offset)) {
|
|
171
|
+
remaining--;
|
|
172
|
+
}
|
|
173
|
+
bit++;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
remaining -= count;
|
|
178
|
+
bit += available;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return bit - 1;
|
|
182
|
+
}
|
|
183
|
+
function readBits(data, byteOffset, bitOffset, count) {
|
|
184
|
+
let value = 0;
|
|
185
|
+
for (let bit = 0; bit < count; bit++) {
|
|
186
|
+
const position = bitOffset + bit;
|
|
187
|
+
if (data.getUint8(byteOffset + (position >>> 3)) & (1 << (position & 7))) {
|
|
188
|
+
value += 2 ** bit;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return value;
|
|
192
|
+
}
|
|
193
|
+
function writeBits(bytes, byteOffset, bitOffset, count, value) {
|
|
194
|
+
for (let bit = 0; bit < count; bit++) {
|
|
195
|
+
if (Math.floor(value / 2 ** bit) % 2) {
|
|
196
|
+
const position = bitOffset + bit;
|
|
197
|
+
const target = byteOffset + (position >>> 3);
|
|
198
|
+
bytes[target] = (bytes[target] ?? 0) | (1 << (position & 7));
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
//# sourceMappingURL=bairro-runs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bairro-runs.js","sourceRoot":"","sources":["../src/bairro-runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,GAGnB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,WAAW,GAAG,CAAC,IAAI,YAAY,CAAC;AACtC,MAAM,QAAQ,GAAG,mBAAmB,EAAE,CAAC;AAcvC,iGAAiG;AACjG,MAAM,UAAU,gBAAgB,CAAC,KAAiB,EAAE,QAAgB,EAAE,OAAkB;IACtF,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3D,IAAI,EAAE,KAAK,QAAQ,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACb,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,SAAS,GAAG,QAAQ,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,YAAY,GAAG,QAAQ,CAAC;IAC3C,MAAM,aAAa,GAAG,UAAU,GAAG,SAAS,CAAC;IAC7C,MAAM,SAAS,GAAG,aAAa,GAAG,WAAW,GAAG,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC/B,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAE1B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,SAAS,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC;QACpD,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC;QAC9C,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,kBAAkB,CAAC,KAAK,EAAE,SAAS,GAAG,KAAK,GAAG,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IACnF,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,IAAI,SAAS,GAAG,WAAW,KAAK,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,KAAK,YAAY,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAC9E,CAAC;YACD,SAAS,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,oBAAoB,CAAC,IAAc,EAAE,MAAoB,EAAE,QAAgB,EAAE,OAAkB;IAC7G,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,SAAS,GAAG,QAAQ,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC;IAC/C,MAAM,UAAU,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IACzE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;AAClH,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,eAAe,CAC7B,IAAc,EACd,MAAoB,EACpB,QAAgB,EAChB,OAAkB,EAClB,WAAmB;IAEnB,IAAI,MAAM,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;QACjC,MAAM,IAAI,4BAA4B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjD,IACE,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,QAAQ,IAAI,OAAO,GAAG,EAAE;WAChD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,YAAY;WACjD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,OAAO;WAC5C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,EACzC,CAAC;QACD,MAAM,IAAI,4BAA4B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,SAAS,GAAG,QAAQ,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC;IAC/C,MAAM,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;IACxC,MAAM,aAAa,GAAG,UAAU,GAAG,SAAS,CAAC;IAC7C,MAAM,SAAS,GAAG,aAAa,GAAG,WAAW,GAAG,CAAC,CAAC;IAClD,IAAI,SAAS,GAAG,QAAQ,GAAG,OAAO,KAAK,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACrE,MAAM,IAAI,4BAA4B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;IACvB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/E,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,IAAI,4BAA4B,CAAC,wCAAwC,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,KAAK,GAAG,WAAW,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,GAAG,CAAC,KAAK,KAAK,YAAY,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;gBAC5G,MAAM,IAAI,4BAA4B,CAAC,yCAAyC,CAAC,CAAC;YACpF,CAAC;YACD,KAAK,EAAE,CAAC;YACR,SAAS;QACX,CAAC;QACD,IAAI,IAAI,IAAI,QAAQ,EAAE,CAAC;YACrB,MAAM,IAAI,4BAA4B,CAAC,wCAAwC,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,GAAG,CAAC;QACpC,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC;QACxE,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,IAAI,aAAa,IAAI,KAAK,IAAI,QAAQ,IAAI,EAAE,GAAG,WAAW,IAAI,EAAE,KAAK,UAAU,EAAE,CAAC;YACxH,MAAM,IAAI,4BAA4B,CAAC,iCAAiC,CAAC,CAAC;QAC5E,CAAC;QACD,aAAa,GAAG,KAAK,CAAC;QACtB,UAAU,GAAG,EAAE,CAAC;QAChB,IAAI,EAAE,CAAC;IACT,CAAC;IACD,IAAI,YAAY,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3E,MAAM,IAAI,4BAA4B,CAAC,yCAAyC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;AAClH,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,kBAAkB,CAAC,IAAc,EAAE,IAAgB,EAAE,GAAW;IAC9E,MAAM,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,GAAG,GAAG,OAAO,CAAC;IAChC,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,IAAI,KAAK,GAAG,GAAG,CAAC;IAChB,OAAO,GAAG,GAAG,KAAK,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClF,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACvB,GAAG,GAAG,MAAM,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,CAAC;QACjB,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,CAAC;IACzB,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,IAAgB,EAAE,SAAiB;IACrE,MAAM,WAAW,GAAG,SAAS,KAAK,YAAY,CAAC;IAC/C,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,GAAG,WAAW,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACrE,IAAI,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,CAAC;IACtD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,GAAG,EAAE,CAAC;IACN,OAAO,SAAS,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,CAAC;QAC3B,MAAM,SAAS,GAAG,CAAC,GAAG,UAAU,CAAC;QACjC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC;QACzF,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACvB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;gBAC7C,IAAI,QAAQ,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC;oBAC7B,SAAS,EAAE,CAAC;gBACd,CAAC;gBACD,GAAG,EAAE,CAAC;YACR,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,IAAI,KAAK,CAAC;YACnB,GAAG,IAAI,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAc,EAAE,UAAkB,EAAE,SAAiB,EAAE,KAAa;IACpF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,SAAS,GAAG,GAAG,CAAC;QACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,KAAK,IAAI,CAAC,IAAI,GAAG,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,KAAiB,EAAE,UAAkB,EAAE,SAAiB,EAAE,KAAa,EAAE,KAAa;IACvG,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,SAAS,GAAG,GAAG,CAAC;YACjC,MAAM,MAAM,GAAG,UAAU,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;YAC7C,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Binary format revision accepted by the current reader and emitted by the writer.
|
|
3
3
|
*/
|
|
4
|
-
export declare const BINARY_DATABASE_VERSION =
|
|
4
|
+
export declare const BINARY_DATABASE_VERSION = 5;
|
|
5
5
|
/**
|
|
6
6
|
* Fixed file-header size in bytes, including the section directory and reserved space.
|
|
7
7
|
*/
|
|
8
8
|
export declare const BINARY_DATABASE_HEADER_SIZE = 256;
|
|
9
|
+
/** SHA-256 footer covering every preceding byte, including header and alignment padding. */
|
|
10
|
+
export declare const BINARY_DATABASE_CHECKSUM_SIZE = 32;
|
|
9
11
|
/** Neighborhood section header: row count and widths of the original DNE identifiers. */
|
|
10
12
|
export declare const BINARY_BAIRRO_HEADER_SIZE = 16;
|
|
11
13
|
/**
|
|
@@ -32,6 +34,12 @@ export declare const CEP_SUFFIX_BITS = 10;
|
|
|
32
34
|
* Size in bytes of a front-coded string dictionary header.
|
|
33
35
|
*/
|
|
34
36
|
export declare const DICTIONARY_HEADER_SIZE = 32;
|
|
37
|
+
/** Dictionary codec identifiers stored at byte 30 of each dictionary header. */
|
|
38
|
+
export declare const DICTIONARY_CODEC_PLAIN = 0;
|
|
39
|
+
export declare const DICTIONARY_CODEC_FSST = 2;
|
|
40
|
+
/** FSST uses 255 symbols of up to eight bytes; code 255 escapes one literal byte. */
|
|
41
|
+
export declare const FSST_SYMBOL_COUNT = 255;
|
|
42
|
+
export declare const FSST_SYMBOL_TABLE_SIZE: number;
|
|
35
43
|
/**
|
|
36
44
|
* Base-two logarithm of the number of strings in one dictionary block.
|
|
37
45
|
*/
|
|
@@ -94,7 +102,7 @@ export type BinaryHeader = {
|
|
|
94
102
|
*/
|
|
95
103
|
municipalityCount: number;
|
|
96
104
|
/**
|
|
97
|
-
* Byte width of municipality IDs in the
|
|
105
|
+
* Byte width of municipality IDs in the locality runs.
|
|
98
106
|
*/
|
|
99
107
|
municipalityIdWidth: ByteWidth;
|
|
100
108
|
/**
|
package/dist/binary-db-format.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Binary format revision accepted by the current reader and emitted by the writer.
|
|
3
3
|
*/
|
|
4
|
-
export const BINARY_DATABASE_VERSION =
|
|
4
|
+
export const BINARY_DATABASE_VERSION = 5;
|
|
5
5
|
/**
|
|
6
6
|
* Fixed file-header size in bytes, including the section directory and reserved space.
|
|
7
7
|
*/
|
|
8
8
|
export const BINARY_DATABASE_HEADER_SIZE = 256;
|
|
9
|
+
/** SHA-256 footer covering every preceding byte, including header and alignment padding. */
|
|
10
|
+
export const BINARY_DATABASE_CHECKSUM_SIZE = 32;
|
|
9
11
|
/** Neighborhood section header: row count and widths of the original DNE identifiers. */
|
|
10
12
|
export const BINARY_BAIRRO_HEADER_SIZE = 16;
|
|
11
13
|
/**
|
|
@@ -32,6 +34,12 @@ export const CEP_SUFFIX_BITS = 10;
|
|
|
32
34
|
* Size in bytes of a front-coded string dictionary header.
|
|
33
35
|
*/
|
|
34
36
|
export const DICTIONARY_HEADER_SIZE = 32;
|
|
37
|
+
/** Dictionary codec identifiers stored at byte 30 of each dictionary header. */
|
|
38
|
+
export const DICTIONARY_CODEC_PLAIN = 0;
|
|
39
|
+
export const DICTIONARY_CODEC_FSST = 2;
|
|
40
|
+
/** FSST uses 255 symbols of up to eight bytes; code 255 escapes one literal byte. */
|
|
41
|
+
export const FSST_SYMBOL_COUNT = 255;
|
|
42
|
+
export const FSST_SYMBOL_TABLE_SIZE = FSST_SYMBOL_COUNT * 9;
|
|
35
43
|
/**
|
|
36
44
|
* Base-two logarithm of the number of strings in one dictionary block.
|
|
37
45
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"binary-db-format.js","sourceRoot":"","sources":["../src/binary-db-format.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AACzC;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC;
|
|
1
|
+
{"version":3,"file":"binary-db-format.js","sourceRoot":"","sources":["../src/binary-db-format.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AACzC;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC;AAC/C,4FAA4F;AAC5F,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEhD,yFAAyF;AACzF,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAE5C;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa;AACnG;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC;AACtC;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,OAAO,CAAC;AACxC;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,GAAG,CAAC,CAAC;AAC7D;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAClC;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AACzC,gFAAgF;AAChF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACxC,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AACvC,qFAAqF;AACrF,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AACrC,MAAM,CAAC,MAAM,sBAAsB,GAAG,iBAAiB,GAAG,CAAC,CAAC;AAC5D;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACxC;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAI,sBAAsB,CAAC;AACjE;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AACnC;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAI,iBAAiB,CAAC;AACvD;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEvC;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,UAAU;IACV,kBAAkB;IAClB,aAAa;IACb,eAAe;IACf,mBAAmB;IACnB,kBAAkB;IAClB,gBAAgB;IAChB,WAAW;IACX,iBAAiB;IACjB,YAAY;IACZ,WAAW;IACX,SAAS;IACT,gBAAgB;IAChB,sBAAsB;IACtB,uBAAuB;IACvB,kBAAkB;IAClB,qBAAqB;IACrB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,oBAAoB;IACpB,cAAc;IACd,2BAA2B;IAC3B,sBAAsB;IACtB,qBAAqB;IACrB,mBAAmB;CACX,CAAC;AAuDX;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa,EAAE,IAAY;IACzD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,IAAY;IACvD,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,WAAW,KAAK,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAgB;IAC/C,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,KAAgB,EAAE,IAAY;IAC/E,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,gBAAgB,KAAK,iCAAiC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAc,EAAE,MAAc,EAAE,KAAgB;IAChF,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACtC,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;kBACxB,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;kBAChC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAkB,EAClB,MAAc,EACd,KAAa,EACb,KAAgB;IAEhB,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;IAClD,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC;IAC9B,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC;IAC5C,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IAC7C,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IAC7C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,IAAI,KAAK,CAAC,UAAU,KAAK,qBAAqB,CAAC,MAAM,EAAE,CAAC;QACtD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,qBAAqB,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC;AAC/E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAkB,EAAE,KAAa,EAAE,KAAa;IAC5E,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,eAAe,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,sBAAsB,eAAe,UAAU,KAAK,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,GAAG,eAAe,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,KAAK,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAG,KAAK,IAAI,KAAK,CAAC;IAC/B,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IAChE,IAAI,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACnF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Fully validates serialized bytes before recording their SHA-256 in the reserved footer. */
|
|
2
|
+
export declare function sealBinaryDatabaseBytes(bytes: Uint8Array): void;
|
|
3
|
+
/** Checks integrity under the trusted producer and immutable file contract. */
|
|
4
|
+
export declare function verifyBinaryDatabaseChecksum(bytes: Uint8Array): void;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { DneBinaryDatabaseFormatError } from "./binary-db-errors.js";
|
|
2
|
+
import { BINARY_DATABASE_CHECKSUM_SIZE } from "./binary-db-format.js";
|
|
3
|
+
import { validateBinaryDatabaseBytes } from "./binary-db-validator.js";
|
|
4
|
+
/** Fully validates serialized bytes before recording their SHA-256 in the reserved footer. */
|
|
5
|
+
export function sealBinaryDatabaseBytes(bytes) {
|
|
6
|
+
validateBinaryDatabaseBytes(bytes);
|
|
7
|
+
const end = bytes.byteLength - BINARY_DATABASE_CHECKSUM_SIZE;
|
|
8
|
+
bytes.set(new Bun.CryptoHasher('sha256').update(bytes.subarray(0, end)).digest(), end);
|
|
9
|
+
}
|
|
10
|
+
/** Checks integrity under the trusted producer and immutable file contract. */
|
|
11
|
+
export function verifyBinaryDatabaseChecksum(bytes) {
|
|
12
|
+
const end = bytes.byteLength - BINARY_DATABASE_CHECKSUM_SIZE;
|
|
13
|
+
const digest = new Bun.CryptoHasher('sha256').update(bytes.subarray(0, end)).digest();
|
|
14
|
+
for (let index = 0; index < BINARY_DATABASE_CHECKSUM_SIZE; index++) {
|
|
15
|
+
if (bytes[end + index] !== digest[index]) {
|
|
16
|
+
throw new DneBinaryDatabaseFormatError('Binary database SHA-256 checksum mismatch');
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=binary-db-integrity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-db-integrity.js","sourceRoot":"","sources":["../src/binary-db-integrity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAE,6BAA6B,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AAEvE,8FAA8F;AAC9F,MAAM,UAAU,uBAAuB,CAAC,KAAiB;IACvD,2BAA2B,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,GAAG,6BAA6B,CAAC;IAC7D,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;AACzF,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,4BAA4B,CAAC,KAAiB;IAC5D,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,GAAG,6BAA6B,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,6BAA6B,EAAE,KAAK,EAAE,EAAE,CAAC;QACnE,IAAI,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,4BAA4B,CAAC,2CAA2C,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type DneBairro, type DneFaixaCep } from './bairro.ts';
|
|
2
|
-
import { type DneRow, type LoadMetadata } from './types.ts';
|
|
2
|
+
import { type DneRow, type LoadMetadata, type UF } from './types.ts';
|
|
3
3
|
/** Error classes and discriminants used by this module's public operations. */
|
|
4
4
|
export { DneBinaryDatabaseClosedError, DneBinaryDatabaseError, type DneBinaryDatabaseErrorCode, DneBinaryDatabaseFormatError, DneBinaryDatabaseIOError, DneBinaryDatabaseVersionError, } from './binary-db-errors.ts';
|
|
5
5
|
/** Header size and format version accepted by this reader. */
|
|
@@ -16,7 +16,8 @@ export declare function isBinaryDatabase(path: string): Promise<boolean>;
|
|
|
16
16
|
/**
|
|
17
17
|
* Provides synchronous, read-only CEP lookups over a memory-mapped DNE binary database.
|
|
18
18
|
*
|
|
19
|
-
* Construction
|
|
19
|
+
* Construction verifies the header and SHA-256 of bytes validated by a trusted generator.
|
|
20
|
+
* Queries decode directly using the invariants guaranteed at generation.
|
|
20
21
|
* Keep the underlying file immutable while the reader is open. Call `close()` when finished.
|
|
21
22
|
* All operational errors extend {@link DneBinaryDatabaseError}; invalid or absent CEPs return `undefined`.
|
|
22
23
|
*/
|
|
@@ -26,16 +27,19 @@ export declare class DneBinaryDatabaseReader {
|
|
|
26
27
|
private decodeScratch;
|
|
27
28
|
private dictionaries;
|
|
28
29
|
private bairros;
|
|
30
|
+
private bairroRuns;
|
|
31
|
+
private localityRuns;
|
|
32
|
+
private logradouroIdBits;
|
|
29
33
|
private header;
|
|
30
34
|
private mapped;
|
|
31
|
-
private metadataValue;
|
|
35
|
+
private readonly metadataValue;
|
|
32
36
|
private municipalityCache;
|
|
33
37
|
/**
|
|
34
|
-
* Opens and
|
|
38
|
+
* Opens and verifies a binary database using Bun's memory-mapping API.
|
|
35
39
|
* @param databasePath - Optional path to an existing binary database. Defaults to `data/dne.bin` inside this package.
|
|
36
40
|
* @throws {DneBinaryDatabaseIOError} If the file cannot be mapped.
|
|
37
41
|
* @throws {DneBinaryDatabaseVersionError} If the declared format version is unsupported.
|
|
38
|
-
* @throws {DneBinaryDatabaseFormatError} If the header,
|
|
42
|
+
* @throws {DneBinaryDatabaseFormatError} If the minimum header, file size, or SHA-256 checksum is invalid.
|
|
39
43
|
*/
|
|
40
44
|
constructor(databasePath?: string);
|
|
41
45
|
/**
|
|
@@ -46,23 +50,35 @@ export declare class DneBinaryDatabaseReader {
|
|
|
46
50
|
close(): void;
|
|
47
51
|
/**
|
|
48
52
|
* Returns the load metadata parsed when the reader was constructed.
|
|
49
|
-
* @returns The cached metadata object
|
|
53
|
+
* @returns The cached, frozen metadata object. Available after `close()`.
|
|
50
54
|
*/
|
|
51
|
-
metadata(): LoadMetadata
|
|
55
|
+
metadata(): Readonly<LoadMetadata>;
|
|
52
56
|
/**
|
|
53
57
|
* Looks up a CEP and decodes its address, including the originating locality indicators.
|
|
54
58
|
* @param cep - Eight ASCII digits or the form `NNNNN-NNN`; surrounding whitespace is not accepted.
|
|
55
59
|
* @returns An address with an eight-digit CEP, or `undefined` when the input is invalid or absent from the database.
|
|
56
60
|
* @throws {DneBinaryDatabaseClosedError} If called after `close()`, including for invalid CEP input.
|
|
57
|
-
* @throws {DneBinaryDatabaseFormatError} If the lookup encounters invalid indexes or record data.
|
|
58
61
|
*/
|
|
59
62
|
queryCep(cep: string): DneRow | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Resolves a municipality's name and UF by its IBGE code.
|
|
65
|
+
* @param municipioCodIbge - Seven-digit IBGE municipality code represented as an integer.
|
|
66
|
+
* @returns The municipality and UF, or undefined for an invalid or unknown code.
|
|
67
|
+
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
68
|
+
*/
|
|
69
|
+
queryMunicipality(municipioCodIbge: number): Pick<DneRow, 'municipio' | 'uf'> | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* Lists the IBGE municipality codes stored for a federative unit.
|
|
72
|
+
* @param uf - Canonical uppercase Brazilian state abbreviation.
|
|
73
|
+
* @returns Unique codes in ascending order, or an empty array for an invalid or absent UF.
|
|
74
|
+
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
75
|
+
*/
|
|
76
|
+
queryMunicipalityCodesByUf(uf: UF): number[];
|
|
60
77
|
/**
|
|
61
78
|
* Reads a neighborhood by its original DNE identifier, independent of duplicate names.
|
|
62
79
|
* @param neighborhoodId - Positive `BAI_NU` identifier.
|
|
63
80
|
* @returns The neighborhood, or undefined for an invalid or unknown identifier.
|
|
64
81
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
65
|
-
* @throws {DneBinaryDatabaseFormatError} If an index, record, or string cannot be decoded.
|
|
66
82
|
*/
|
|
67
83
|
queryNeighborhood(neighborhoodId: number): DneBairro | undefined;
|
|
68
84
|
/**
|
|
@@ -70,7 +86,6 @@ export declare class DneBinaryDatabaseReader {
|
|
|
70
86
|
* @param cep - Eight ASCII digits or `NNNNN-NNN`.
|
|
71
87
|
* @returns The neighborhood, or undefined for an invalid, unknown, or neighborhood-free CEP.
|
|
72
88
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
73
|
-
* @throws {DneBinaryDatabaseFormatError} If the CEP or neighborhood data is invalid.
|
|
74
89
|
*/
|
|
75
90
|
queryNeighborhoodByCep(cep: string): DneBairro | undefined;
|
|
76
91
|
/**
|
|
@@ -78,7 +93,6 @@ export declare class DneBinaryDatabaseReader {
|
|
|
78
93
|
* @param neighborhoodId - Positive `BAI_NU` identifier.
|
|
79
94
|
* @returns Intervals sorted by lower and upper bounds, or an empty array for an invalid, unknown, or rangeless neighborhood.
|
|
80
95
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
81
|
-
* @throws {DneBinaryDatabaseFormatError} If the interval directory or bounds are invalid.
|
|
82
96
|
*/
|
|
83
97
|
queryNeighborhoodCepRanges(neighborhoodId: number): DneFaixaCep[];
|
|
84
98
|
/**
|
|
@@ -86,7 +100,7 @@ export declare class DneBinaryDatabaseReader {
|
|
|
86
100
|
* @returns The row count recorded at construction; available after `close()`.
|
|
87
101
|
*/
|
|
88
102
|
rowCount(): number;
|
|
89
|
-
private
|
|
103
|
+
private assertOpen;
|
|
90
104
|
private findCepIndex;
|
|
91
105
|
private findNeighborhoodIndex;
|
|
92
106
|
private readNeighborhoodIndex;
|
|
@@ -100,15 +114,13 @@ export declare class DneBinaryDatabaseReader {
|
|
|
100
114
|
private readPacked10;
|
|
101
115
|
private readPackedInteger;
|
|
102
116
|
private readUint32;
|
|
103
|
-
private requireData;
|
|
104
|
-
private requireMapped;
|
|
105
117
|
}
|
|
106
118
|
/**
|
|
107
119
|
* Opens a binary database, reads its load metadata, and closes the temporary reader.
|
|
108
120
|
* @param path - Path to an existing binary database file.
|
|
109
|
-
* @returns
|
|
121
|
+
* @returns Frozen metadata parsed from the file at open time.
|
|
110
122
|
* @throws {DneBinaryDatabaseIOError} If the file cannot be mapped.
|
|
111
123
|
* @throws {DneBinaryDatabaseVersionError} If the binary version is unsupported.
|
|
112
|
-
* @throws {DneBinaryDatabaseFormatError} If the file
|
|
124
|
+
* @throws {DneBinaryDatabaseFormatError} If the minimum header, file size, or SHA-256 checksum is invalid.
|
|
113
125
|
*/
|
|
114
|
-
export declare function readBinaryDatabaseMetadata(path: string): Promise<LoadMetadata
|
|
126
|
+
export declare function readBinaryDatabaseMetadata(path: string): Promise<Readonly<LoadMetadata>>;
|