@konstit/dne 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -351
- package/data/dne.bin +0 -0
- package/dist/bairro-runs.d.ts +20 -0
- package/dist/bairro-runs.js +202 -0
- package/dist/bairro-runs.js.map +1 -0
- package/dist/binary-db-format.d.ts +10 -2
- package/dist/binary-db-format.js +9 -1
- package/dist/binary-db-format.js.map +1 -1
- package/dist/binary-db-integrity.d.ts +4 -0
- package/dist/binary-db-integrity.js +20 -0
- package/dist/binary-db-integrity.js.map +1 -0
- package/dist/binary-db-reader.d.ts +29 -17
- package/dist/binary-db-reader.js +191 -335
- package/dist/binary-db-reader.js.map +1 -1
- package/dist/binary-db-validator.d.ts +13 -0
- package/dist/binary-db-validator.js +712 -0
- package/dist/binary-db-validator.js.map +1 -0
- package/dist/fsst.d.ts +18 -0
- package/dist/fsst.js +117 -0
- package/dist/fsst.js.map +1 -0
- package/dist/index.js +44 -44
- package/dist/index.js.map +16 -9
- package/dist/locality-runs.d.ts +18 -0
- package/dist/locality-runs.js +108 -0
- package/dist/locality-runs.js.map +1 -0
- package/dist/packed-bits.d.ts +6 -0
- package/dist/packed-bits.js +41 -0
- package/dist/packed-bits.js.map +1 -0
- package/package.json +2 -2
package/dist/binary-db-reader.js
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import { fileURLToPath } from 'node:url';
|
|
2
|
+
import { readBairroRunIndex, readBairroRunsLayout, } from "./bairro-runs.js";
|
|
2
3
|
import { isBairroId, } from "./bairro.js";
|
|
3
4
|
import { DneBinaryDatabaseClosedError, DneBinaryDatabaseError, DneBinaryDatabaseFormatError, DneBinaryDatabaseIOError, DneBinaryDatabaseVersionError, } from "./binary-db-errors.js";
|
|
4
|
-
import { BINARY_BAIRRO_HEADER_SIZE, BINARY_DATABASE_HEADER_SIZE, BINARY_DATABASE_MAGIC as MAGIC, BINARY_DATABASE_VERSION, BINARY_SECTION_NAMES as SECTION_NAMES,
|
|
5
|
+
import { BINARY_BAIRRO_HEADER_SIZE, BINARY_DATABASE_CHECKSUM_SIZE, BINARY_DATABASE_HEADER_SIZE, BINARY_DATABASE_MAGIC as MAGIC, BINARY_DATABASE_VERSION, BINARY_SECTION_NAMES as SECTION_NAMES, CEP_SUFFIX_BITS, createPopcountTable, DICTIONARY_BLOCK_SHIFT, DICTIONARY_CODEC_FSST, hasBinaryMagic as hasMagic, integerByteWidth, readPackedInteger as readPackedIntegerFromData, SECTION_TABLE_OFFSET, SPARSE_RANK_ROWS, SPARSE_RANK_SHIFT, } from "./binary-db-format.js";
|
|
6
|
+
import { verifyBinaryDatabaseChecksum } from "./binary-db-integrity.js";
|
|
5
7
|
import { cepToU32 } from "./cep.js";
|
|
8
|
+
import { readTrustedFsstString } from "./fsst.js";
|
|
9
|
+
import { findLocalityRun, readLocalityRunsLayout, } from "./locality-runs.js";
|
|
10
|
+
import { integerBitWidth, readPackedBits, } from "./packed-bits.js";
|
|
6
11
|
import { LOCALIDADE_SITUACOES, LOCALIDADE_TIPOS, } from "./schema.js";
|
|
7
|
-
import {
|
|
12
|
+
import { stateCodes, } from "./types.js";
|
|
8
13
|
/** Error classes and discriminants used by this module's public operations. */
|
|
9
14
|
export { DneBinaryDatabaseClosedError, DneBinaryDatabaseError, DneBinaryDatabaseFormatError, DneBinaryDatabaseIOError, DneBinaryDatabaseVersionError, } from "./binary-db-errors.js";
|
|
10
15
|
/** Header size and format version accepted by this reader. */
|
|
@@ -32,7 +37,8 @@ export async function isBinaryDatabase(path) {
|
|
|
32
37
|
/**
|
|
33
38
|
* Provides synchronous, read-only CEP lookups over a memory-mapped DNE binary database.
|
|
34
39
|
*
|
|
35
|
-
* Construction
|
|
40
|
+
* Construction verifies the header and SHA-256 of bytes validated by a trusted generator.
|
|
41
|
+
* Queries decode directly using the invariants guaranteed at generation.
|
|
36
42
|
* Keep the underlying file immutable while the reader is open. Call `close()` when finished.
|
|
37
43
|
* All operational errors extend {@link DneBinaryDatabaseError}; invalid or absent CEPs return `undefined`.
|
|
38
44
|
*/
|
|
@@ -42,16 +48,19 @@ export class DneBinaryDatabaseReader {
|
|
|
42
48
|
decodeScratch = new Uint8Array(0xff);
|
|
43
49
|
dictionaries;
|
|
44
50
|
bairros;
|
|
51
|
+
bairroRuns;
|
|
52
|
+
localityRuns;
|
|
53
|
+
logradouroIdBits;
|
|
45
54
|
header;
|
|
46
55
|
mapped;
|
|
47
56
|
metadataValue;
|
|
48
57
|
municipalityCache;
|
|
49
58
|
/**
|
|
50
|
-
* Opens and
|
|
59
|
+
* Opens and verifies a binary database using Bun's memory-mapping API.
|
|
51
60
|
* @param databasePath - Optional path to an existing binary database. Defaults to `data/dne.bin` inside this package.
|
|
52
61
|
* @throws {DneBinaryDatabaseIOError} If the file cannot be mapped.
|
|
53
62
|
* @throws {DneBinaryDatabaseVersionError} If the declared format version is unsupported.
|
|
54
|
-
* @throws {DneBinaryDatabaseFormatError} If the header,
|
|
63
|
+
* @throws {DneBinaryDatabaseFormatError} If the minimum header, file size, or SHA-256 checksum is invalid.
|
|
55
64
|
*/
|
|
56
65
|
constructor(databasePath = fileURLToPath(new URL('../data/dne.bin', import.meta.url))) {
|
|
57
66
|
let mapped;
|
|
@@ -63,18 +72,23 @@ export class DneBinaryDatabaseReader {
|
|
|
63
72
|
}
|
|
64
73
|
try {
|
|
65
74
|
const data = new DataView(mapped.buffer, mapped.byteOffset, mapped.byteLength);
|
|
75
|
+
verifyHeader(data);
|
|
76
|
+
verifyBinaryDatabaseChecksum(mapped);
|
|
66
77
|
const header = readHeader(data);
|
|
67
|
-
validateHeader(data, header);
|
|
68
78
|
const dictionaries = readDictionaries(data, header);
|
|
69
79
|
const bairros = readNeighborhoods(data, header, dictionaries);
|
|
70
|
-
|
|
80
|
+
const bairroRuns = readBairroRunsLayout(data, header.sections.bairroIds, header.rowCount, bairros.idWidth);
|
|
81
|
+
const localityRuns = readLocalityRunsLayout(data, header.sections.municipalityIds, header.rowCount);
|
|
71
82
|
const metadata = parseMetadata(mapped, header.sections.metadata);
|
|
72
83
|
this.mapped = mapped;
|
|
73
84
|
this.data = data;
|
|
74
85
|
this.header = header;
|
|
75
86
|
this.dictionaries = dictionaries;
|
|
76
87
|
this.bairros = bairros;
|
|
77
|
-
this.
|
|
88
|
+
this.bairroRuns = bairroRuns;
|
|
89
|
+
this.localityRuns = localityRuns;
|
|
90
|
+
this.logradouroIdBits = integerBitWidth(dictionaries.logradouro.count);
|
|
91
|
+
this.metadataValue = Object.freeze(metadata);
|
|
78
92
|
this.bairroCache = Array.from({ length: dictionaries.bairro.count + 1 });
|
|
79
93
|
this.municipalityCache = Array.from({ length: header.municipalityCount + 1 });
|
|
80
94
|
}
|
|
@@ -100,7 +114,7 @@ export class DneBinaryDatabaseReader {
|
|
|
100
114
|
}
|
|
101
115
|
/**
|
|
102
116
|
* Returns the load metadata parsed when the reader was constructed.
|
|
103
|
-
* @returns The cached metadata object
|
|
117
|
+
* @returns The cached, frozen metadata object. Available after `close()`.
|
|
104
118
|
*/
|
|
105
119
|
metadata() {
|
|
106
120
|
return this.metadataValue;
|
|
@@ -110,74 +124,127 @@ export class DneBinaryDatabaseReader {
|
|
|
110
124
|
* @param cep - Eight ASCII digits or the form `NNNNN-NNN`; surrounding whitespace is not accepted.
|
|
111
125
|
* @returns An address with an eight-digit CEP, or `undefined` when the input is invalid or absent from the database.
|
|
112
126
|
* @throws {DneBinaryDatabaseClosedError} If called after `close()`, including for invalid CEP input.
|
|
113
|
-
* @throws {DneBinaryDatabaseFormatError} If the lookup encounters invalid indexes or record data.
|
|
114
127
|
*/
|
|
115
128
|
queryCep(cep) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
129
|
+
this.assertOpen();
|
|
130
|
+
const index = this.findCepIndex(cep);
|
|
131
|
+
return index === -1 ? undefined : this.readRow(index, cep.replace('-', ''));
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Resolves a municipality's name and UF by its IBGE code.
|
|
135
|
+
* @param municipioCodIbge - Seven-digit IBGE municipality code represented as an integer.
|
|
136
|
+
* @returns The municipality and UF, or undefined for an invalid or unknown code.
|
|
137
|
+
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
138
|
+
*/
|
|
139
|
+
queryMunicipality(municipioCodIbge) {
|
|
140
|
+
this.assertOpen();
|
|
141
|
+
if (!Number.isInteger(municipioCodIbge) || municipioCodIbge < 1_000_000 || municipioCodIbge > 9_999_999) {
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
const recordWidth = 3 + this.dictionaries.municipio.idWidth + this.dictionaries.uf.idWidth;
|
|
145
|
+
const offset = this.header.sections.municipalities.offset;
|
|
146
|
+
let low = 0;
|
|
147
|
+
let high = this.header.municipalityCount - 1;
|
|
148
|
+
while (low <= high) {
|
|
149
|
+
const middle = low + ((high - low) >>> 1);
|
|
150
|
+
const code = this.readPackedInteger(offset + middle * recordWidth, 3);
|
|
151
|
+
if (code < municipioCodIbge) {
|
|
152
|
+
low = middle + 1;
|
|
153
|
+
}
|
|
154
|
+
else if (code > municipioCodIbge) {
|
|
155
|
+
high = middle - 1;
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
const { municipio, uf } = this.readMunicipality(middle + 1);
|
|
159
|
+
return { municipio, uf };
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return undefined;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Lists the IBGE municipality codes stored for a federative unit.
|
|
166
|
+
* @param uf - Canonical uppercase Brazilian state abbreviation.
|
|
167
|
+
* @returns Unique codes in ascending order, or an empty array for an invalid or absent UF.
|
|
168
|
+
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
169
|
+
*/
|
|
170
|
+
queryMunicipalityCodesByUf(uf) {
|
|
171
|
+
this.assertOpen();
|
|
172
|
+
if (!Object.hasOwn(stateCodes, uf)) {
|
|
173
|
+
return [];
|
|
174
|
+
}
|
|
175
|
+
const minimum = stateCodes[uf] * 100_000;
|
|
176
|
+
const maximum = minimum + 100_000;
|
|
177
|
+
const recordWidth = 3 + this.dictionaries.municipio.idWidth + this.dictionaries.uf.idWidth;
|
|
178
|
+
const offset = this.header.sections.municipalities.offset;
|
|
179
|
+
let low = 0;
|
|
180
|
+
let high = this.header.municipalityCount;
|
|
181
|
+
while (low < high) {
|
|
182
|
+
const middle = low + ((high - low) >>> 1);
|
|
183
|
+
if (this.readPackedInteger(offset + middle * recordWidth, 3) < minimum) {
|
|
184
|
+
low = middle + 1;
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
high = middle;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
const codes = [];
|
|
191
|
+
for (let index = low; index < this.header.municipalityCount; index++) {
|
|
192
|
+
const code = this.readPackedInteger(offset + index * recordWidth, 3);
|
|
193
|
+
if (code >= maximum) {
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
codes.push(code);
|
|
197
|
+
}
|
|
198
|
+
return codes;
|
|
120
199
|
}
|
|
121
200
|
/**
|
|
122
201
|
* Reads a neighborhood by its original DNE identifier, independent of duplicate names.
|
|
123
202
|
* @param neighborhoodId - Positive `BAI_NU` identifier.
|
|
124
203
|
* @returns The neighborhood, or undefined for an invalid or unknown identifier.
|
|
125
204
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
126
|
-
* @throws {DneBinaryDatabaseFormatError} If an index, record, or string cannot be decoded.
|
|
127
205
|
*/
|
|
128
206
|
queryNeighborhood(neighborhoodId) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
});
|
|
207
|
+
this.assertOpen();
|
|
208
|
+
const index = this.findNeighborhoodIndex(neighborhoodId);
|
|
209
|
+
return index === 0 ? undefined : this.readNeighborhood(index);
|
|
133
210
|
}
|
|
134
211
|
/**
|
|
135
212
|
* Resolves the actual neighborhood of a CEP, preserving the distinction from districts and villages.
|
|
136
213
|
* @param cep - Eight ASCII digits or `NNNNN-NNN`.
|
|
137
214
|
* @returns The neighborhood, or undefined for an invalid, unknown, or neighborhood-free CEP.
|
|
138
215
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
139
|
-
* @throws {DneBinaryDatabaseFormatError} If the CEP or neighborhood data is invalid.
|
|
140
216
|
*/
|
|
141
217
|
queryNeighborhoodByCep(cep) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
});
|
|
218
|
+
this.assertOpen();
|
|
219
|
+
const row = this.findCepIndex(cep);
|
|
220
|
+
if (row === -1) {
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
const index = this.readNeighborhoodIndex(row);
|
|
224
|
+
return index === 0 ? undefined : this.readNeighborhood(index);
|
|
150
225
|
}
|
|
151
226
|
/**
|
|
152
227
|
* Reads a neighborhood's original inclusive CEP intervals without merging gaps.
|
|
153
228
|
* @param neighborhoodId - Positive `BAI_NU` identifier.
|
|
154
229
|
* @returns Intervals sorted by lower and upper bounds, or an empty array for an invalid, unknown, or rangeless neighborhood.
|
|
155
230
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
156
|
-
* @throws {DneBinaryDatabaseFormatError} If the interval directory or bounds are invalid.
|
|
157
231
|
*/
|
|
158
232
|
queryNeighborhoodCepRanges(neighborhoodId) {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (initial > final || final > 99_999_999) {
|
|
175
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood CEP range');
|
|
176
|
-
}
|
|
177
|
-
result.push({ cep_inicial: String(initial).padStart(8, '0'), cep_final: String(final).padStart(8, '0') });
|
|
178
|
-
}
|
|
179
|
-
return result;
|
|
180
|
-
});
|
|
233
|
+
this.assertOpen();
|
|
234
|
+
const index = this.findNeighborhoodIndex(neighborhoodId);
|
|
235
|
+
if (index === 0) {
|
|
236
|
+
return [];
|
|
237
|
+
}
|
|
238
|
+
const { bairroFaixaOffsets: offsets, bairroFaixas: ranges } = this.header.sections;
|
|
239
|
+
const start = this.readUint32(offsets.offset + (index - 1) * 4);
|
|
240
|
+
const end = this.readUint32(offsets.offset + index * 4);
|
|
241
|
+
const result = [];
|
|
242
|
+
for (let row = start; row < end; row++) {
|
|
243
|
+
const initial = this.readUint32(ranges.offset + row * 8);
|
|
244
|
+
const final = this.readUint32(ranges.offset + row * 8 + 4);
|
|
245
|
+
result.push({ cep_inicial: String(initial).padStart(8, '0'), cep_final: String(final).padStart(8, '0') });
|
|
246
|
+
}
|
|
247
|
+
return result;
|
|
181
248
|
}
|
|
182
249
|
/**
|
|
183
250
|
* Returns the number of CEP records declared by the validated header.
|
|
@@ -186,16 +253,9 @@ export class DneBinaryDatabaseReader {
|
|
|
186
253
|
rowCount() {
|
|
187
254
|
return this.header.rowCount;
|
|
188
255
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
return read();
|
|
193
|
-
}
|
|
194
|
-
catch (error) {
|
|
195
|
-
if (error instanceof DneBinaryDatabaseError) {
|
|
196
|
-
throw error;
|
|
197
|
-
}
|
|
198
|
-
throw new DneBinaryDatabaseFormatError('Unable to decode binary database record', { cause: error });
|
|
256
|
+
assertOpen() {
|
|
257
|
+
if (this.data === null) {
|
|
258
|
+
throw new DneBinaryDatabaseClosedError();
|
|
199
259
|
}
|
|
200
260
|
}
|
|
201
261
|
findCepIndex(cep) {
|
|
@@ -210,9 +270,6 @@ export class DneBinaryDatabaseReader {
|
|
|
210
270
|
let low = this.readPackedInteger(prefixRegion.offset + prefix * width, width);
|
|
211
271
|
const prefixEnd = this.readPackedInteger(prefixRegion.offset + (prefix + 1) * width, width);
|
|
212
272
|
let high = prefixEnd;
|
|
213
|
-
if (low > prefixEnd || prefixEnd > this.header.rowCount) {
|
|
214
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary CEP prefix range: ${prefix}`);
|
|
215
|
-
}
|
|
216
273
|
while (low < high) {
|
|
217
274
|
const middle = low + ((high - low) >>> 1);
|
|
218
275
|
const current = this.readPacked10(middle);
|
|
@@ -251,11 +308,7 @@ export class DneBinaryDatabaseReader {
|
|
|
251
308
|
return id === neighborhoodId ? low + 1 : 0;
|
|
252
309
|
}
|
|
253
310
|
readNeighborhoodIndex(row) {
|
|
254
|
-
|
|
255
|
-
if (index > this.bairros.count) {
|
|
256
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary neighborhood index: ${index}`);
|
|
257
|
-
}
|
|
258
|
-
return index;
|
|
311
|
+
return readBairroRunIndex(this.data, this.bairroRuns, row);
|
|
259
312
|
}
|
|
260
313
|
readNeighborhood(index) {
|
|
261
314
|
const { originalIdWidth, localidadeIdWidth, recordsOffset, recordWidth } = this.bairros;
|
|
@@ -264,45 +317,34 @@ export class DneBinaryDatabaseReader {
|
|
|
264
317
|
offset += originalIdWidth;
|
|
265
318
|
const localidade_id = this.readPackedInteger(offset, localidadeIdWidth);
|
|
266
319
|
offset += localidadeIdWidth;
|
|
267
|
-
const nome = this.readRequiredDictionaryString(this.dictionaries.bairro, this.readPackedInteger(offset, this.dictionaries.bairro.idWidth)
|
|
320
|
+
const nome = this.readRequiredDictionaryString(this.dictionaries.bairro, this.readPackedInteger(offset, this.dictionaries.bairro.idWidth));
|
|
268
321
|
offset += this.dictionaries.bairro.idWidth;
|
|
269
322
|
const nome_abreviado = this.readDictionaryString(this.dictionaries.bairroAbreviado, this.readPackedInteger(offset, this.dictionaries.bairroAbreviado.idWidth));
|
|
270
323
|
offset += this.dictionaries.bairroAbreviado.idWidth;
|
|
271
|
-
const uf =
|
|
324
|
+
const uf = this.readRequiredDictionaryString(this.dictionaries.uf, this.readPackedInteger(offset, this.dictionaries.uf.idWidth));
|
|
272
325
|
return { bairro_id, localidade_id, nome, nome_abreviado, uf };
|
|
273
326
|
}
|
|
274
327
|
readRow(index, cep) {
|
|
275
328
|
const sections = this.header.sections;
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary municipality id: ${municipalityId}`);
|
|
279
|
-
}
|
|
329
|
+
const run = findLocalityRun(this.data, this.localityRuns, index);
|
|
330
|
+
const municipalityId = this.readPackedInteger(this.localityRuns.idsOffset + run * this.header.municipalityIdWidth, this.header.municipalityIdWidth);
|
|
280
331
|
const municipality = this.readMunicipality(municipalityId);
|
|
281
|
-
const flags = this.
|
|
332
|
+
const flags = this.data.getUint8(sections.localidadeFlags.offset + run);
|
|
282
333
|
const situacao = LOCALIDADE_SITUACOES[flags & 3];
|
|
283
334
|
const tipo = LOCALIDADE_TIPOS[flags >>> 2];
|
|
284
|
-
if (situacao === undefined || tipo === undefined) {
|
|
285
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary locality indicators: ${flags}`);
|
|
286
|
-
}
|
|
287
335
|
const bairroIndex = this.readNeighborhoodIndex(index);
|
|
288
336
|
const localidadeNomeId = this.readSparseId(sections.localidadeNomeBitmap, sections.localidadeNomeRanks, sections.localidadeNomeIds, index, this.dictionaries.bairro.idWidth);
|
|
289
|
-
if (localidadeNomeId !== 0 && (bairroIndex !== 0 || tipo === 'municipio')) {
|
|
290
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary subordinate locality name');
|
|
291
|
-
}
|
|
292
337
|
const bairroNameId = bairroIndex === 0
|
|
293
338
|
? localidadeNomeId
|
|
294
339
|
: this.readPackedInteger(this.bairros.recordsOffset + (bairroIndex - 1) * this.bairros.recordWidth
|
|
295
340
|
+ this.bairros.originalIdWidth + this.bairros.localidadeIdWidth, this.dictionaries.bairro.idWidth);
|
|
296
|
-
if (bairroIndex !== 0 && bairroNameId === 0) {
|
|
297
|
-
throw new DneBinaryDatabaseFormatError('Null binary neighborhood name');
|
|
298
|
-
}
|
|
299
341
|
return {
|
|
300
342
|
bairro: this.readCachedDictionaryString(this.dictionaries.bairro, bairroNameId, this.bairroCache),
|
|
301
343
|
cep,
|
|
302
344
|
complemento: this.readDictionaryString(this.dictionaries.complemento, this.readSparseId(sections.complementoBitmap, sections.complementoRanks, sections.complementoIds, index, this.dictionaries.complemento.idWidth)),
|
|
303
345
|
localidade_situacao: situacao,
|
|
304
346
|
localidade_tipo: tipo,
|
|
305
|
-
logradouro: this.readDictionaryString(this.dictionaries.logradouro, this.
|
|
347
|
+
logradouro: this.readDictionaryString(this.dictionaries.logradouro, readPackedBits(this.data, sections.logradouroIds.offset, index, this.logradouroIdBits)),
|
|
306
348
|
municipio: municipality.municipio,
|
|
307
349
|
municipio_cod_ibge: municipality.codigoIbge,
|
|
308
350
|
nome: this.readDictionaryString(this.dictionaries.nome, this.readSparseId(sections.nomeBitmap, sections.nomeRanks, sections.nomeIds, index, this.dictionaries.nome.idWidth)),
|
|
@@ -320,16 +362,16 @@ export class DneBinaryDatabaseReader {
|
|
|
320
362
|
const ufId = this.readPackedInteger(offset + 3 + this.dictionaries.municipio.idWidth, this.dictionaries.uf.idWidth);
|
|
321
363
|
const municipality = {
|
|
322
364
|
codigoIbge: this.readPackedInteger(offset, 3),
|
|
323
|
-
municipio: this.readRequiredDictionaryString(this.dictionaries.municipio, municipioId
|
|
324
|
-
uf:
|
|
365
|
+
municipio: this.readRequiredDictionaryString(this.dictionaries.municipio, municipioId),
|
|
366
|
+
uf: this.readRequiredDictionaryString(this.dictionaries.uf, ufId),
|
|
325
367
|
};
|
|
326
368
|
this.municipalityCache[id] = municipality;
|
|
327
369
|
return municipality;
|
|
328
370
|
}
|
|
329
371
|
readSparseId(bitmap, ranks, ids, rowIndex, idWidth) {
|
|
330
|
-
const mapped = this.
|
|
372
|
+
const mapped = this.mapped;
|
|
331
373
|
const bitmapByteIndex = rowIndex >>> 3;
|
|
332
|
-
const bitmapByte = mapped[bitmap.offset + bitmapByteIndex]
|
|
374
|
+
const bitmapByte = mapped[bitmap.offset + bitmapByteIndex];
|
|
333
375
|
const bitIndex = rowIndex & 7;
|
|
334
376
|
if ((bitmapByte & (1 << bitIndex)) === 0) {
|
|
335
377
|
return 0;
|
|
@@ -338,42 +380,30 @@ export class DneBinaryDatabaseReader {
|
|
|
338
380
|
let denseIndex = this.readUint32(ranks.offset + rankBlock * 4);
|
|
339
381
|
const blockByteOffset = rankBlock * (SPARSE_RANK_ROWS >>> 3);
|
|
340
382
|
for (let offset = blockByteOffset; offset < bitmapByteIndex; offset++) {
|
|
341
|
-
denseIndex += popcount[mapped[bitmap.offset + offset]
|
|
342
|
-
}
|
|
343
|
-
denseIndex += popcount[bitmapByte & ((1 << bitIndex) - 1)] ?? 0;
|
|
344
|
-
if ((denseIndex + 1) * idWidth > ids.length) {
|
|
345
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary sparse id index: ${denseIndex}`);
|
|
383
|
+
denseIndex += popcount[mapped[bitmap.offset + offset]];
|
|
346
384
|
}
|
|
385
|
+
denseIndex += popcount[bitmapByte & ((1 << bitIndex) - 1)];
|
|
347
386
|
return this.readPackedInteger(ids.offset + denseIndex * idWidth, idWidth);
|
|
348
387
|
}
|
|
349
388
|
readDictionaryString(dictionary, id) {
|
|
350
389
|
if (id === 0) {
|
|
351
390
|
return null;
|
|
352
391
|
}
|
|
353
|
-
|
|
354
|
-
|
|
392
|
+
const mapped = this.mapped;
|
|
393
|
+
if (dictionary.codec === DICTIONARY_CODEC_FSST) {
|
|
394
|
+
return readTrustedFsstString(mapped, this.data, dictionary, id);
|
|
355
395
|
}
|
|
356
|
-
const mapped = this.requireMapped();
|
|
357
396
|
const index = id - 1;
|
|
358
397
|
const block = index >>> DICTIONARY_BLOCK_SHIFT;
|
|
359
398
|
const blockStart = block << DICTIONARY_BLOCK_SHIFT;
|
|
360
399
|
const relativeOffset = this.readUint32(dictionary.blockOffsetsOffset + block * 4);
|
|
361
|
-
if (relativeOffset > dictionary.suffixDataLength) {
|
|
362
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary string block offset: ${relativeOffset}`);
|
|
363
|
-
}
|
|
364
400
|
let cursor = dictionary.suffixDataOffset + relativeOffset;
|
|
365
401
|
const decoded = this.decodeScratch;
|
|
366
402
|
let previousLength = 0;
|
|
367
403
|
for (let currentIndex = blockStart; currentIndex <= index; currentIndex++) {
|
|
368
|
-
const length = mapped[dictionary.lengthsOffset + currentIndex]
|
|
369
|
-
const prefixLength = mapped[dictionary.prefixesOffset + currentIndex]
|
|
370
|
-
if ((currentIndex === blockStart && prefixLength !== 0) || prefixLength > previousLength || prefixLength > length) {
|
|
371
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary front-coded string: ${id}`);
|
|
372
|
-
}
|
|
404
|
+
const length = mapped[dictionary.lengthsOffset + currentIndex];
|
|
405
|
+
const prefixLength = mapped[dictionary.prefixesOffset + currentIndex];
|
|
373
406
|
const suffixLength = length - prefixLength;
|
|
374
|
-
if (cursor < dictionary.suffixDataOffset || cursor + suffixLength > dictionary.suffixDataOffset + dictionary.suffixDataLength) {
|
|
375
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary string suffix: ${id}`);
|
|
376
|
-
}
|
|
377
407
|
decoded.set(mapped.subarray(cursor, cursor + suffixLength), prefixLength);
|
|
378
408
|
cursor += suffixLength;
|
|
379
409
|
previousLength = length;
|
|
@@ -388,57 +418,39 @@ export class DneBinaryDatabaseReader {
|
|
|
388
418
|
if (cached !== undefined) {
|
|
389
419
|
return cached;
|
|
390
420
|
}
|
|
391
|
-
const value = this.
|
|
392
|
-
|
|
393
|
-
cache[id] = value;
|
|
394
|
-
}
|
|
421
|
+
const value = this.readRequiredDictionaryString(dictionary, id);
|
|
422
|
+
cache[id] = value;
|
|
395
423
|
return value;
|
|
396
424
|
}
|
|
397
|
-
readRequiredDictionaryString(dictionary, id
|
|
398
|
-
|
|
399
|
-
if (value === null) {
|
|
400
|
-
throw new DneBinaryDatabaseFormatError(`Binary database has a null required field: ${field}`);
|
|
401
|
-
}
|
|
402
|
-
return value;
|
|
425
|
+
readRequiredDictionaryString(dictionary, id) {
|
|
426
|
+
return this.readDictionaryString(dictionary, id);
|
|
403
427
|
}
|
|
404
428
|
readPacked10(index) {
|
|
405
|
-
const mapped = this.
|
|
429
|
+
const mapped = this.mapped;
|
|
406
430
|
const region = this.header.sections.cepSuffixes;
|
|
407
431
|
const bitOffset = index * CEP_SUFFIX_BITS;
|
|
408
432
|
const byteIndex = bitOffset >>> 3;
|
|
409
433
|
const shift = bitOffset & 7;
|
|
410
434
|
const absoluteOffset = region.offset + byteIndex;
|
|
411
|
-
const value =
|
|
412
|
-
| (
|
|
413
|
-
| (
|
|
435
|
+
const value = mapped[absoluteOffset]
|
|
436
|
+
| (mapped[absoluteOffset + 1] << 8)
|
|
437
|
+
| (mapped[absoluteOffset + 2] << 16);
|
|
414
438
|
return (value >>> shift) & 0x3ff;
|
|
415
439
|
}
|
|
416
440
|
readPackedInteger(offset, width) {
|
|
417
|
-
return readPackedIntegerFromData(this.
|
|
441
|
+
return readPackedIntegerFromData(this.data, offset, width);
|
|
418
442
|
}
|
|
419
443
|
readUint32(offset) {
|
|
420
|
-
return this.
|
|
421
|
-
}
|
|
422
|
-
requireData() {
|
|
423
|
-
if (!this.data) {
|
|
424
|
-
throw new DneBinaryDatabaseClosedError();
|
|
425
|
-
}
|
|
426
|
-
return this.data;
|
|
427
|
-
}
|
|
428
|
-
requireMapped() {
|
|
429
|
-
if (!this.mapped) {
|
|
430
|
-
throw new DneBinaryDatabaseClosedError();
|
|
431
|
-
}
|
|
432
|
-
return this.mapped;
|
|
444
|
+
return this.data.getUint32(offset, true);
|
|
433
445
|
}
|
|
434
446
|
}
|
|
435
447
|
/**
|
|
436
448
|
* Opens a binary database, reads its load metadata, and closes the temporary reader.
|
|
437
449
|
* @param path - Path to an existing binary database file.
|
|
438
|
-
* @returns
|
|
450
|
+
* @returns Frozen metadata parsed from the file at open time.
|
|
439
451
|
* @throws {DneBinaryDatabaseIOError} If the file cannot be mapped.
|
|
440
452
|
* @throws {DneBinaryDatabaseVersionError} If the binary version is unsupported.
|
|
441
|
-
* @throws {DneBinaryDatabaseFormatError} If the file
|
|
453
|
+
* @throws {DneBinaryDatabaseFormatError} If the minimum header, file size, or SHA-256 checksum is invalid.
|
|
442
454
|
*/
|
|
443
455
|
export async function readBinaryDatabaseMetadata(path) {
|
|
444
456
|
const reader = new DneBinaryDatabaseReader(path);
|
|
@@ -449,8 +461,8 @@ export async function readBinaryDatabaseMetadata(path) {
|
|
|
449
461
|
reader.close();
|
|
450
462
|
}
|
|
451
463
|
}
|
|
452
|
-
function
|
|
453
|
-
if (data.byteLength < BINARY_DATABASE_HEADER_SIZE) {
|
|
464
|
+
function verifyHeader(data) {
|
|
465
|
+
if (data.byteLength < BINARY_DATABASE_HEADER_SIZE + BINARY_DATABASE_CHECKSUM_SIZE) {
|
|
454
466
|
throw new DneBinaryDatabaseFormatError('Binary database header is truncated');
|
|
455
467
|
}
|
|
456
468
|
for (let index = 0; index < MAGIC.length; index++) {
|
|
@@ -471,6 +483,11 @@ function readHeader(data) {
|
|
|
471
483
|
if (data.getUint16(24, true) !== SECTION_NAMES.length) {
|
|
472
484
|
throw new DneBinaryDatabaseFormatError('Unsupported binary database section table');
|
|
473
485
|
}
|
|
486
|
+
if (data.getUint32(16, true) !== data.byteLength) {
|
|
487
|
+
throw new DneBinaryDatabaseFormatError(`Binary database size mismatch: header=${data.getUint32(16, true)}, actual=${data.byteLength}`);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
function readHeader(data) {
|
|
474
491
|
const sections = {};
|
|
475
492
|
for (const [index, name,] of SECTION_NAMES.entries()) {
|
|
476
493
|
const offset = SECTION_TABLE_OFFSET + index * 8;
|
|
@@ -480,221 +497,60 @@ function readHeader(data) {
|
|
|
480
497
|
};
|
|
481
498
|
}
|
|
482
499
|
return {
|
|
483
|
-
cepPrefixOffsetWidth:
|
|
500
|
+
cepPrefixOffsetWidth: data.getUint8(20),
|
|
484
501
|
fileSize: data.getUint32(16, true),
|
|
485
502
|
municipalityCount: data.getUint32(28, true),
|
|
486
|
-
municipalityIdWidth:
|
|
503
|
+
municipalityIdWidth: data.getUint8(21),
|
|
487
504
|
rowCount: data.getUint32(12, true),
|
|
488
505
|
sections,
|
|
489
506
|
};
|
|
490
507
|
}
|
|
491
|
-
function validateHeader(data, header) {
|
|
492
|
-
if (header.fileSize !== data.byteLength) {
|
|
493
|
-
throw new DneBinaryDatabaseFormatError(`Binary database size mismatch: header=${header.fileSize}, actual=${data.byteLength}`);
|
|
494
|
-
}
|
|
495
|
-
if (!header.rowCount || !header.municipalityCount) {
|
|
496
|
-
throw new DneBinaryDatabaseFormatError('Binary database must contain rows and municipalities');
|
|
497
|
-
}
|
|
498
|
-
if (header.rowCount > maxPackedInteger(header.cepPrefixOffsetWidth)) {
|
|
499
|
-
throw new DneBinaryDatabaseFormatError('Binary row count does not fit the CEP prefix offset width');
|
|
500
|
-
}
|
|
501
|
-
if (header.municipalityCount > maxPackedInteger(header.municipalityIdWidth)) {
|
|
502
|
-
throw new DneBinaryDatabaseFormatError('Binary municipality count does not fit its id width');
|
|
503
|
-
}
|
|
504
|
-
let previousEnd = BINARY_DATABASE_HEADER_SIZE;
|
|
505
|
-
for (const name of SECTION_NAMES) {
|
|
506
|
-
const region = header.sections[name];
|
|
507
|
-
validateRegion(data.byteLength, region, name);
|
|
508
|
-
if (region.offset < previousEnd) {
|
|
509
|
-
throw new DneBinaryDatabaseFormatError(`Binary section is out of order or overlaps: ${name}`);
|
|
510
|
-
}
|
|
511
|
-
previousEnd = region.offset + region.length;
|
|
512
|
-
}
|
|
513
|
-
if (header.sections.cepPrefixOffsets.length !== CEP_PREFIX_OFFSETS_COUNT * header.cepPrefixOffsetWidth) {
|
|
514
|
-
throw new DneBinaryDatabaseFormatError('Binary CEP prefix directory has an invalid length');
|
|
515
|
-
}
|
|
516
|
-
if (header.sections.cepSuffixes.length !== packedBitLength(header.rowCount, CEP_SUFFIX_BITS)) {
|
|
517
|
-
throw new DneBinaryDatabaseFormatError('Binary CEP suffix column has an invalid length');
|
|
518
|
-
}
|
|
519
|
-
if (header.sections.complementoBitmap.length !== bitmapByteLength(header.rowCount)
|
|
520
|
-
|| header.sections.nomeBitmap.length !== bitmapByteLength(header.rowCount)
|
|
521
|
-
|| header.sections.localidadeNomeBitmap.length !== bitmapByteLength(header.rowCount)) {
|
|
522
|
-
throw new DneBinaryDatabaseFormatError('Binary nullable bitmap has an invalid length');
|
|
523
|
-
}
|
|
524
|
-
const expectedRankLength = (Math.ceil(header.rowCount / SPARSE_RANK_ROWS) + 1) * 4;
|
|
525
|
-
if (header.sections.complementoRanks.length !== expectedRankLength
|
|
526
|
-
|| header.sections.nomeRanks.length !== expectedRankLength
|
|
527
|
-
|| header.sections.localidadeNomeRanks.length !== expectedRankLength) {
|
|
528
|
-
throw new DneBinaryDatabaseFormatError('Binary sparse rank index has an invalid length');
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
508
|
function readDictionaries(data, header) {
|
|
532
509
|
return {
|
|
533
|
-
bairro: readDictionary(data, header.sections.bairroDictionary
|
|
534
|
-
bairroAbreviado: readDictionary(data, header.sections.bairroAbreviadoDictionary
|
|
535
|
-
complemento: readDictionary(data, header.sections.complementoDictionary
|
|
536
|
-
logradouro: readDictionary(data, header.sections.logradouroDictionary
|
|
537
|
-
municipio: readDictionary(data, header.sections.municipioDictionary
|
|
538
|
-
nome: readDictionary(data, header.sections.nomeDictionary
|
|
539
|
-
uf: readDictionary(data, header.sections.ufDictionary
|
|
510
|
+
bairro: readDictionary(data, header.sections.bairroDictionary),
|
|
511
|
+
bairroAbreviado: readDictionary(data, header.sections.bairroAbreviadoDictionary),
|
|
512
|
+
complemento: readDictionary(data, header.sections.complementoDictionary),
|
|
513
|
+
logradouro: readDictionary(data, header.sections.logradouroDictionary),
|
|
514
|
+
municipio: readDictionary(data, header.sections.municipioDictionary),
|
|
515
|
+
nome: readDictionary(data, header.sections.nomeDictionary),
|
|
516
|
+
uf: readDictionary(data, header.sections.ufDictionary),
|
|
540
517
|
};
|
|
541
518
|
}
|
|
542
|
-
function readDictionary(data, region
|
|
543
|
-
if (region.length < DICTIONARY_HEADER_SIZE) {
|
|
544
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary is truncated`);
|
|
545
|
-
}
|
|
519
|
+
function readDictionary(data, region) {
|
|
546
520
|
const count = data.getUint32(region.offset, true);
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
const suffixDataRelative = data.getUint32(region.offset + 20, true);
|
|
552
|
-
const suffixDataLength = data.getUint32(region.offset + 24, true);
|
|
553
|
-
const idWidth = readByteWidth(data.getUint8(region.offset + 28), `${name} dictionary id`);
|
|
554
|
-
if (data.getUint8(region.offset + 29) !== DICTIONARY_BLOCK_SHIFT) {
|
|
555
|
-
throw new DneBinaryDatabaseFormatError(`Unsupported binary ${name} dictionary block size`);
|
|
556
|
-
}
|
|
557
|
-
if (blockCount !== Math.ceil(count / DICTIONARY_BLOCK_SIZE)) {
|
|
558
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid block count`);
|
|
559
|
-
}
|
|
560
|
-
if (count > maxPackedInteger(idWidth)) {
|
|
561
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary does not fit its id width`);
|
|
562
|
-
}
|
|
563
|
-
if (blockOffsetsRelative !== DICTIONARY_HEADER_SIZE
|
|
564
|
-
|| lengthsRelative !== blockOffsetsRelative + blockCount * 4
|
|
565
|
-
|| prefixesRelative !== lengthsRelative + count
|
|
566
|
-
|| suffixDataRelative !== prefixesRelative + count
|
|
567
|
-
|| suffixDataRelative > region.length
|
|
568
|
-
|| suffixDataLength !== region.length - suffixDataRelative) {
|
|
569
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid layout`);
|
|
570
|
-
}
|
|
571
|
-
const dictionary = {
|
|
572
|
-
blockCount,
|
|
573
|
-
blockOffsetsOffset: region.offset + blockOffsetsRelative,
|
|
521
|
+
const prefixesOffset = region.offset + data.getUint32(region.offset + 16, true);
|
|
522
|
+
return {
|
|
523
|
+
blockCount: data.getUint32(region.offset + 4, true),
|
|
524
|
+
blockOffsetsOffset: region.offset + data.getUint32(region.offset + 8, true),
|
|
574
525
|
count,
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
526
|
+
codec: data.getUint8(region.offset + 30),
|
|
527
|
+
idWidth: data.getUint8(region.offset + 28),
|
|
528
|
+
lengthsOffset: region.offset + data.getUint32(region.offset + 12, true),
|
|
529
|
+
lengthWidth: data.getUint8(region.offset + 31) === 2 ? 2 : 1,
|
|
530
|
+
prefixesOffset,
|
|
578
531
|
region,
|
|
579
|
-
suffixDataLength,
|
|
580
|
-
suffixDataOffset: region.offset +
|
|
532
|
+
suffixDataLength: data.getUint32(region.offset + 24, true),
|
|
533
|
+
suffixDataOffset: region.offset + data.getUint32(region.offset + 20, true),
|
|
534
|
+
symbolsOffset: prefixesOffset + count,
|
|
581
535
|
};
|
|
582
|
-
if (blockCount && data.getUint32(dictionary.blockOffsetsOffset, true) !== 0) {
|
|
583
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid first block offset`);
|
|
584
|
-
}
|
|
585
|
-
return dictionary;
|
|
586
536
|
}
|
|
587
537
|
function readNeighborhoods(data, header, dictionaries) {
|
|
588
538
|
const region = header.sections.bairros;
|
|
589
|
-
if (region.length < BINARY_BAIRRO_HEADER_SIZE) {
|
|
590
|
-
throw new DneBinaryDatabaseFormatError('Binary neighborhood table is truncated');
|
|
591
|
-
}
|
|
592
539
|
const count = data.getUint32(region.offset, true);
|
|
593
|
-
const originalIdWidth =
|
|
594
|
-
const localidadeIdWidth =
|
|
540
|
+
const originalIdWidth = data.getUint8(region.offset + 4);
|
|
541
|
+
const localidadeIdWidth = data.getUint8(region.offset + 5);
|
|
595
542
|
const recordWidth = originalIdWidth + localidadeIdWidth + dictionaries.bairro.idWidth
|
|
596
543
|
+ dictionaries.bairroAbreviado.idWidth + dictionaries.uf.idWidth;
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
offset += localidadeIdWidth;
|
|
606
|
-
const nameId = readPackedIntegerFromData(data, offset, dictionaries.bairro.idWidth);
|
|
607
|
-
offset += dictionaries.bairro.idWidth;
|
|
608
|
-
const abbreviationId = readPackedIntegerFromData(data, offset, dictionaries.bairroAbreviado.idWidth);
|
|
609
|
-
offset += dictionaries.bairroAbreviado.idWidth;
|
|
610
|
-
const ufId = readPackedIntegerFromData(data, offset, dictionaries.uf.idWidth);
|
|
611
|
-
if (id <= previousId || localityId === 0 || nameId === 0 || nameId > dictionaries.bairro.count
|
|
612
|
-
|| abbreviationId > dictionaries.bairroAbreviado.count || ufId === 0 || ufId > dictionaries.uf.count) {
|
|
613
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary neighborhood record: ${index}`);
|
|
614
|
-
}
|
|
615
|
-
previousId = id;
|
|
616
|
-
}
|
|
617
|
-
return { count, idWidth: integerByteWidth(count), originalIdWidth, localidadeIdWidth, recordsOffset, recordWidth };
|
|
618
|
-
}
|
|
619
|
-
function validateNeighborhoodRanges(data, header, bairros) {
|
|
620
|
-
const { bairroFaixaOffsets: offsets, bairroFaixas: ranges } = header.sections;
|
|
621
|
-
assertRegionLength(offsets, (bairros.count + 1) * 4, 'neighborhood range offsets');
|
|
622
|
-
if (ranges.length % 8 !== 0 || data.getUint32(offsets.offset, true) !== 0) {
|
|
623
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood range layout');
|
|
624
|
-
}
|
|
625
|
-
const count = ranges.length / 8;
|
|
626
|
-
let start = 0;
|
|
627
|
-
for (let index = 0; index < bairros.count; index++) {
|
|
628
|
-
const end = data.getUint32(offsets.offset + (index + 1) * 4, true);
|
|
629
|
-
if (end < start || end > count) {
|
|
630
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood range offsets');
|
|
631
|
-
}
|
|
632
|
-
let previousInitial = -1;
|
|
633
|
-
let previousFinal = -1;
|
|
634
|
-
for (let row = start; row < end; row++) {
|
|
635
|
-
const initial = data.getUint32(ranges.offset + row * 8, true);
|
|
636
|
-
const final = data.getUint32(ranges.offset + row * 8 + 4, true);
|
|
637
|
-
if (initial > final || final > 99_999_999 || initial < previousInitial
|
|
638
|
-
|| (initial === previousInitial && final <= previousFinal)) {
|
|
639
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood CEP range');
|
|
640
|
-
}
|
|
641
|
-
previousInitial = initial;
|
|
642
|
-
previousFinal = final;
|
|
643
|
-
}
|
|
644
|
-
start = end;
|
|
645
|
-
}
|
|
646
|
-
if (start !== count) {
|
|
647
|
-
throw new DneBinaryDatabaseFormatError('Unreferenced binary neighborhood CEP ranges');
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
function validateLayout(data, header, dictionaries, bairros) {
|
|
651
|
-
const sections = header.sections;
|
|
652
|
-
assertRegionLength(sections.localidadeFlags, header.rowCount, 'locality indicators');
|
|
653
|
-
assertRegionLength(sections.logradouroIds, header.rowCount * dictionaries.logradouro.idWidth, 'logradouro ids');
|
|
654
|
-
assertRegionLength(sections.bairroIds, header.rowCount * bairros.idWidth, 'bairro indexes');
|
|
655
|
-
assertRegionLength(sections.municipalityIds, header.rowCount * header.municipalityIdWidth, 'municipality ids');
|
|
656
|
-
const municipalityRecordWidth = 3 + dictionaries.municipio.idWidth + dictionaries.uf.idWidth;
|
|
657
|
-
assertRegionLength(sections.municipalities, header.municipalityCount * municipalityRecordWidth, 'municipalities');
|
|
658
|
-
validateSparseLayout(data, sections.complementoRanks, sections.complementoIds, dictionaries.complemento.idWidth, 'complemento');
|
|
659
|
-
validateSparseLayout(data, sections.nomeRanks, sections.nomeIds, dictionaries.nome.idWidth, 'nome');
|
|
660
|
-
validateSparseLayout(data, sections.localidadeNomeRanks, sections.localidadeNomeIds, dictionaries.bairro.idWidth, 'localidade_nome');
|
|
661
|
-
validateNeighborhoodRanges(data, header, bairros);
|
|
662
|
-
const firstPrefix = readPackedIntegerFromData(data, sections.cepPrefixOffsets.offset, header.cepPrefixOffsetWidth);
|
|
663
|
-
const lastPrefix = readPackedIntegerFromData(data, sections.cepPrefixOffsets.offset + CEP_PREFIX_COUNT * header.cepPrefixOffsetWidth, header.cepPrefixOffsetWidth);
|
|
664
|
-
if (firstPrefix !== 0 || lastPrefix !== header.rowCount) {
|
|
665
|
-
throw new DneBinaryDatabaseFormatError('Binary CEP prefix directory has invalid boundaries');
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
function validateSparseLayout(data, ranks, ids, idWidth, name) {
|
|
669
|
-
const denseCount = data.getUint32(ranks.offset + ranks.length - 4, true);
|
|
670
|
-
if (ids.length !== denseCount * idWidth) {
|
|
671
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} sparse ids have an invalid length`);
|
|
672
|
-
}
|
|
544
|
+
return {
|
|
545
|
+
count,
|
|
546
|
+
idWidth: integerByteWidth(count),
|
|
547
|
+
originalIdWidth,
|
|
548
|
+
localidadeIdWidth,
|
|
549
|
+
recordsOffset: region.offset + BINARY_BAIRRO_HEADER_SIZE,
|
|
550
|
+
recordWidth,
|
|
551
|
+
};
|
|
673
552
|
}
|
|
674
553
|
function parseMetadata(mapped, region) {
|
|
675
|
-
|
|
676
|
-
let parsed;
|
|
677
|
-
try {
|
|
678
|
-
parsed = JSON.parse(text);
|
|
679
|
-
}
|
|
680
|
-
catch (cause) {
|
|
681
|
-
throw new DneBinaryDatabaseFormatError('Binary database metadata is invalid JSON', { cause });
|
|
682
|
-
}
|
|
683
|
-
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
684
|
-
throw new DneBinaryDatabaseFormatError('Binary database metadata must be an object');
|
|
685
|
-
}
|
|
686
|
-
return parsed;
|
|
687
|
-
}
|
|
688
|
-
function validateRegion(fileLength, region, name) {
|
|
689
|
-
if (region.offset < BINARY_DATABASE_HEADER_SIZE
|
|
690
|
-
|| region.offset > fileLength
|
|
691
|
-
|| region.length > fileLength - region.offset) {
|
|
692
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} region is outside the file`);
|
|
693
|
-
}
|
|
694
|
-
}
|
|
695
|
-
function assertRegionLength(region, expected, name) {
|
|
696
|
-
if (region.length !== expected) {
|
|
697
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} region has an invalid length`);
|
|
698
|
-
}
|
|
554
|
+
return JSON.parse(textDecoder.decode(mapped.subarray(region.offset, region.offset + region.length)));
|
|
699
555
|
}
|
|
700
556
|
//# sourceMappingURL=binary-db-reader.js.map
|