@konstit/dne 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,712 @@
1
+ import { parseBairroRuns, } from "./bairro-runs.js";
2
+ import { DneBinaryDatabaseFormatError } from "./binary-db-errors.js";
3
+ import { alignBinaryOffset, BINARY_BAIRRO_HEADER_SIZE, BINARY_DATABASE_CHECKSUM_SIZE, BINARY_DATABASE_HEADER_SIZE, BINARY_DATABASE_MAGIC as MAGIC, BINARY_DATABASE_VERSION, BINARY_SECTION_NAMES as SECTION_NAMES, bitmapByteLength, CEP_PREFIX_COUNT, CEP_PREFIX_OFFSETS_COUNT, CEP_SUFFIX_BITS, DICTIONARY_BLOCK_SHIFT, DICTIONARY_BLOCK_SIZE, DICTIONARY_CODEC_FSST, DICTIONARY_CODEC_PLAIN, DICTIONARY_HEADER_SIZE, FSST_SYMBOL_COUNT, FSST_SYMBOL_TABLE_SIZE, integerByteWidth, maxPackedInteger, packedBitLength, readByteWidth, readPackedInteger, SECTION_TABLE_OFFSET, SPARSE_RANK_ROWS, SPARSE_RANK_SHIFT, } from "./binary-db-format.js";
4
+ import { parseLocalityRuns, } from "./locality-runs.js";
5
+ import { integerBitWidth, readPackedBits, } from "./packed-bits.js";
6
+ import { LOCALIDADE_SITUACOES, LOCALIDADE_TIPOS, } from "./schema.js";
7
+ import { parseUF, stateCodes, } from "./types.js";
8
+ const textDecoder = new TextDecoder('utf-8', { fatal: true });
9
+ /**
10
+ * Validates a complete serialized v5 database before its checksum is sealed.
11
+ *
12
+ * The final {@link BINARY_DATABASE_CHECKSUM_SIZE} bytes are reserved for the
13
+ * SHA-256 footer and are intentionally ignored here. The caller may pass the
14
+ * zeroed pre-seal buffer or an already sealed buffer. All bytes before that
15
+ * footer are checked, including the canonical section padding.
16
+ *
17
+ * @param bytes - Complete file bytes, including the reserved checksum footer.
18
+ * @throws {DneBinaryDatabaseFormatError} If any header, section, index,
19
+ * dictionary, reference, or cross-field invariant is violated.
20
+ */
21
+ export function validateBinaryDatabaseBytes(bytes) {
22
+ try {
23
+ if (!(bytes instanceof Uint8Array)) {
24
+ throw new DneBinaryDatabaseFormatError('Binary database bytes must be a Uint8Array');
25
+ }
26
+ const data = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
27
+ const header = readHeader(data);
28
+ validateHeader(bytes, header);
29
+ const dictionaries = readDictionaries(bytes, data, header);
30
+ validateUfDictionary(dictionaries.uf);
31
+ const bairros = readNeighborhoods(bytes, data, header, dictionaries);
32
+ const municipalityTable = validateMunicipalities(data, header, dictionaries);
33
+ const sparse = validateSparseColumns(bytes, data, header, dictionaries);
34
+ const bairroRuns = parseBairroRuns(data, header.sections.bairroIds, header.rowCount, bairros.idWidth, bairros.count);
35
+ const localityRuns = parseLocalityRuns(data, header.sections.municipalityIds, header.rowCount, header.municipalityIdWidth);
36
+ validateRunSections(data, header, bairroRuns, localityRuns);
37
+ validateNeighborhoodRanges(data, header, bairros);
38
+ validateCepColumns(bytes, data, header);
39
+ validateLogradouroIds(bytes, data, header, dictionaries.logradouro);
40
+ validateRows(data, header, dictionaries, bairros, bairroRuns, localityRuns, sparse, municipalityTable);
41
+ validateMetadata(bytes, header.sections.metadata);
42
+ }
43
+ catch (error) {
44
+ if (error instanceof DneBinaryDatabaseFormatError) {
45
+ throw error;
46
+ }
47
+ const message = error instanceof Error ? error.message : String(error);
48
+ throw new DneBinaryDatabaseFormatError(`Unable to validate binary database: ${message}`, { cause: error });
49
+ }
50
+ }
51
+ function readHeader(data) {
52
+ if (data.byteLength < BINARY_DATABASE_HEADER_SIZE + BINARY_DATABASE_CHECKSUM_SIZE) {
53
+ throw new DneBinaryDatabaseFormatError('Binary database header is truncated');
54
+ }
55
+ for (let index = 0; index < MAGIC.length; index++) {
56
+ if (data.getUint8(index) !== MAGIC[index]) {
57
+ throw new DneBinaryDatabaseFormatError('Invalid binary database magic');
58
+ }
59
+ }
60
+ if (data.getUint16(8, true) !== BINARY_DATABASE_VERSION) {
61
+ throw new DneBinaryDatabaseFormatError('Unsupported binary database version');
62
+ }
63
+ if (data.getUint16(10, true) !== BINARY_DATABASE_HEADER_SIZE) {
64
+ throw new DneBinaryDatabaseFormatError('Unsupported binary database header size');
65
+ }
66
+ if (data.getUint8(22) !== SPARSE_RANK_SHIFT || data.getUint8(23) !== DICTIONARY_BLOCK_SHIFT) {
67
+ throw new DneBinaryDatabaseFormatError('Unsupported binary database block layout');
68
+ }
69
+ if (data.getUint16(24, true) !== SECTION_NAMES.length) {
70
+ throw new DneBinaryDatabaseFormatError('Unsupported binary database section table');
71
+ }
72
+ const sections = {};
73
+ for (const [index, name,] of SECTION_NAMES.entries()) {
74
+ const offset = SECTION_TABLE_OFFSET + index * 8;
75
+ sections[name] = {
76
+ length: data.getUint32(offset + 4, true),
77
+ offset: data.getUint32(offset, true),
78
+ };
79
+ }
80
+ return {
81
+ cepPrefixOffsetWidth: readByteWidth(data.getUint8(20), 'CEP prefix offset'),
82
+ fileSize: data.getUint32(16, true),
83
+ municipalityCount: data.getUint32(28, true),
84
+ municipalityIdWidth: readByteWidth(data.getUint8(21), 'municipality id'),
85
+ rowCount: data.getUint32(12, true),
86
+ sections,
87
+ };
88
+ }
89
+ function validateHeader(bytes, header) {
90
+ const payloadEnd = bytes.byteLength - BINARY_DATABASE_CHECKSUM_SIZE;
91
+ if (header.fileSize !== bytes.byteLength) {
92
+ throw new DneBinaryDatabaseFormatError(`Binary database size mismatch: header=${header.fileSize}, actual=${bytes.byteLength}`);
93
+ }
94
+ if (payloadEnd < BINARY_DATABASE_HEADER_SIZE || payloadEnd % 8 !== 0) {
95
+ throw new DneBinaryDatabaseFormatError('Binary database payload has an invalid alignment');
96
+ }
97
+ if (!header.rowCount || !header.municipalityCount) {
98
+ throw new DneBinaryDatabaseFormatError('Binary database must contain rows and municipalities');
99
+ }
100
+ if (header.rowCount > maxPackedInteger(header.cepPrefixOffsetWidth)) {
101
+ throw new DneBinaryDatabaseFormatError('Binary row count does not fit the CEP prefix offset width');
102
+ }
103
+ if (header.municipalityCount > maxPackedInteger(header.municipalityIdWidth)) {
104
+ throw new DneBinaryDatabaseFormatError('Binary municipality count does not fit its id width');
105
+ }
106
+ if (header.cepPrefixOffsetWidth !== integerByteWidth(header.rowCount)
107
+ || header.municipalityIdWidth !== integerByteWidth(header.municipalityCount)) {
108
+ throw new DneBinaryDatabaseFormatError('Binary database uses a noncanonical integer width');
109
+ }
110
+ validateZeroRange(bytes, 26, 28, 'binary header reserved bytes');
111
+ validateZeroRange(bytes, 248, BINARY_DATABASE_HEADER_SIZE, 'binary header reserved bytes');
112
+ let previousEnd = BINARY_DATABASE_HEADER_SIZE;
113
+ for (const name of SECTION_NAMES) {
114
+ const region = header.sections[name];
115
+ validateRegion(payloadEnd, region, name);
116
+ if (region.offset % 8 !== 0 || region.offset < previousEnd) {
117
+ throw new DneBinaryDatabaseFormatError(`Binary section is out of order or unaligned: ${name}`);
118
+ }
119
+ validateZeroRange(bytes, previousEnd, region.offset, `${name} section padding`);
120
+ previousEnd = region.offset + region.length;
121
+ }
122
+ validateZeroRange(bytes, previousEnd, payloadEnd, 'final binary section padding');
123
+ if (alignBinaryOffset(previousEnd) !== payloadEnd) {
124
+ throw new DneBinaryDatabaseFormatError('Binary database footer is not after aligned section data');
125
+ }
126
+ assertRegionLength(header.sections.cepPrefixOffsets, CEP_PREFIX_OFFSETS_COUNT * header.cepPrefixOffsetWidth, 'CEP prefix directory');
127
+ assertRegionLength(header.sections.cepSuffixes, packedBitLength(header.rowCount, CEP_SUFFIX_BITS), 'CEP suffix column');
128
+ for (const name of ['complementoBitmap', 'nomeBitmap', 'localidadeNomeBitmap']) {
129
+ assertRegionLength(header.sections[name], bitmapByteLength(header.rowCount), 'nullable bitmap');
130
+ }
131
+ const expectedRankLength = (Math.ceil(header.rowCount / SPARSE_RANK_ROWS) + 1) * 4;
132
+ for (const name of ['complementoRanks', 'nomeRanks', 'localidadeNomeRanks']) {
133
+ assertRegionLength(header.sections[name], expectedRankLength, 'sparse rank index');
134
+ }
135
+ }
136
+ function readDictionaries(bytes, data, header) {
137
+ return {
138
+ bairro: readDictionary(bytes, data, header.sections.bairroDictionary, 'bairro', false),
139
+ bairroAbreviado: readDictionary(bytes, data, header.sections.bairroAbreviadoDictionary, 'bairro abbreviation', false),
140
+ complemento: readDictionary(bytes, data, header.sections.complementoDictionary, 'complemento', false),
141
+ logradouro: readDictionary(bytes, data, header.sections.logradouroDictionary, 'logradouro', true),
142
+ municipio: readDictionary(bytes, data, header.sections.municipioDictionary, 'municipio', false),
143
+ nome: readDictionary(bytes, data, header.sections.nomeDictionary, 'nome', false),
144
+ uf: readDictionary(bytes, data, header.sections.ufDictionary, 'uf', false),
145
+ };
146
+ }
147
+ function validateUfDictionary(dictionary) {
148
+ for (const [index, value,] of dictionary.values.entries()) {
149
+ requireCanonicalUf(value, index);
150
+ }
151
+ }
152
+ function requireCanonicalUf(value, index) {
153
+ let parsed;
154
+ try {
155
+ parsed = parseUF(value);
156
+ }
157
+ catch (cause) {
158
+ throw new DneBinaryDatabaseFormatError(`Invalid binary UF dictionary entry: ${index}`, { cause });
159
+ }
160
+ if (value !== parsed) {
161
+ throw new DneBinaryDatabaseFormatError(`Binary UF dictionary entry is not canonical: ${index}`);
162
+ }
163
+ return parsed;
164
+ }
165
+ function readDictionary(bytes, data, region, name, allowFsst) {
166
+ if (region.length < DICTIONARY_HEADER_SIZE) {
167
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary is truncated`);
168
+ }
169
+ const count = data.getUint32(region.offset, true);
170
+ const blockCount = data.getUint32(region.offset + 4, true);
171
+ const blockOffsetsRelative = data.getUint32(region.offset + 8, true);
172
+ const lengthsRelative = data.getUint32(region.offset + 12, true);
173
+ const prefixesRelative = data.getUint32(region.offset + 16, true);
174
+ const suffixDataRelative = data.getUint32(region.offset + 20, true);
175
+ const suffixDataLength = data.getUint32(region.offset + 24, true);
176
+ const idWidth = readByteWidth(data.getUint8(region.offset + 28), `${name} dictionary id`);
177
+ const codec = data.getUint8(region.offset + 30);
178
+ const encodedWidth = data.getUint8(region.offset + 31);
179
+ if ((codec !== DICTIONARY_CODEC_PLAIN && (codec !== DICTIONARY_CODEC_FSST || !allowFsst))
180
+ || (codec === DICTIONARY_CODEC_PLAIN ? encodedWidth !== 0 : encodedWidth !== 1 && encodedWidth !== 2)) {
181
+ throw new DneBinaryDatabaseFormatError(`Unsupported binary ${name} dictionary codec or length width`);
182
+ }
183
+ if (data.getUint8(region.offset + 29) !== DICTIONARY_BLOCK_SHIFT) {
184
+ throw new DneBinaryDatabaseFormatError(`Unsupported binary ${name} dictionary block size`);
185
+ }
186
+ const expectedBlockCount = Math.ceil(count / DICTIONARY_BLOCK_SIZE);
187
+ if (blockCount !== expectedBlockCount || count > maxPackedInteger(idWidth)) {
188
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid count or id width`);
189
+ }
190
+ const lengthWidth = encodedWidth === 2 ? 2 : 1;
191
+ const symbolTableSize = codec === DICTIONARY_CODEC_FSST ? FSST_SYMBOL_TABLE_SIZE : 0;
192
+ if (blockOffsetsRelative !== DICTIONARY_HEADER_SIZE
193
+ || lengthsRelative !== blockOffsetsRelative + blockCount * 4
194
+ || prefixesRelative !== lengthsRelative + count * lengthWidth
195
+ || suffixDataRelative !== prefixesRelative + count + symbolTableSize
196
+ || suffixDataRelative > region.length
197
+ || suffixDataLength !== region.length - suffixDataRelative
198
+ || idWidth !== integerByteWidth(count)) {
199
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid layout`);
200
+ }
201
+ const dictionary = {
202
+ blockCount,
203
+ blockOffsetsOffset: region.offset + blockOffsetsRelative,
204
+ codec,
205
+ count,
206
+ idWidth,
207
+ lengthWidth,
208
+ lengthsOffset: region.offset + lengthsRelative,
209
+ prefixesOffset: region.offset + prefixesRelative,
210
+ region,
211
+ suffixDataLength,
212
+ suffixDataOffset: region.offset + suffixDataRelative,
213
+ symbolsOffset: region.offset + prefixesRelative + count,
214
+ values: [],
215
+ };
216
+ validateDictionaryBlocks(bytes, data, dictionary, name);
217
+ dictionary.values = decodeDictionary(bytes, data, dictionary, name);
218
+ return dictionary;
219
+ }
220
+ function validateDictionaryBlocks(bytes, data, dictionary, name) {
221
+ if (dictionary.blockCount && data.getUint32(dictionary.blockOffsetsOffset, true) !== 0) {
222
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid first block offset`);
223
+ }
224
+ let previous = 0;
225
+ for (let block = 0; block < dictionary.blockCount; block++) {
226
+ const offset = data.getUint32(dictionary.blockOffsetsOffset + block * 4, true);
227
+ if (offset < previous || offset > dictionary.suffixDataLength) {
228
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has invalid block offsets`);
229
+ }
230
+ previous = offset;
231
+ }
232
+ if (dictionary.codec === DICTIONARY_CODEC_FSST) {
233
+ for (let code = 0; code < FSST_SYMBOL_COUNT; code++) {
234
+ const length = bytes[dictionary.symbolsOffset + code] ?? 0;
235
+ if (length > 8) {
236
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} FSST symbol length`);
237
+ }
238
+ const slot = dictionary.symbolsOffset + FSST_SYMBOL_COUNT + code * 8;
239
+ validateZeroRange(bytes, slot + length, slot + 8, `binary ${name} FSST symbol padding`);
240
+ }
241
+ }
242
+ if (dictionary.blockCount === 0 && dictionary.suffixDataLength !== 0) {
243
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has unexpected payload`);
244
+ }
245
+ }
246
+ function decodeDictionary(bytes, data, dictionary, name) {
247
+ const values = [];
248
+ const retainValues = name === 'bairro' || name === 'municipio' || name === 'uf';
249
+ let previousValue;
250
+ for (let block = 0; block < dictionary.blockCount; block++) {
251
+ const blockStart = block * DICTIONARY_BLOCK_SIZE;
252
+ const blockEndIndex = Math.min(dictionary.count, blockStart + DICTIONARY_BLOCK_SIZE);
253
+ const relative = data.getUint32(dictionary.blockOffsetsOffset + block * 4, true);
254
+ const nextRelative = block + 1 < dictionary.blockCount
255
+ ? data.getUint32(dictionary.blockOffsetsOffset + (block + 1) * 4, true)
256
+ : dictionary.suffixDataLength;
257
+ let cursor = dictionary.suffixDataOffset + relative;
258
+ const blockEnd = dictionary.suffixDataOffset + nextRelative;
259
+ let blockPrevious = new Uint8Array();
260
+ for (let index = blockStart; index < blockEndIndex; index++) {
261
+ const prefix = bytes[dictionary.prefixesOffset + index] ?? 0;
262
+ if ((index === blockStart && prefix !== 0) || prefix > blockPrevious.byteLength) {
263
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} dictionary prefix: ${index}`);
264
+ }
265
+ let current;
266
+ if (dictionary.codec === DICTIONARY_CODEC_PLAIN) {
267
+ const length = bytes[dictionary.lengthsOffset + index] ?? 0;
268
+ if (prefix > length) {
269
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} dictionary length: ${index}`);
270
+ }
271
+ const suffixLength = length - prefix;
272
+ if (cursor < dictionary.suffixDataOffset || cursor + suffixLength > blockEnd) {
273
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} dictionary suffix: ${index}`);
274
+ }
275
+ current = new Uint8Array(length);
276
+ current.set(blockPrevious.subarray(0, prefix));
277
+ current.set(bytes.subarray(cursor, cursor + suffixLength), prefix);
278
+ cursor += suffixLength;
279
+ }
280
+ else {
281
+ const compressedLength = dictionary.lengthWidth === 1
282
+ ? bytes[dictionary.lengthsOffset + index] ?? 0
283
+ : data.getUint16(dictionary.lengthsOffset + index * 2, true);
284
+ if (cursor < dictionary.suffixDataOffset || cursor + compressedLength > blockEnd) {
285
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} FSST suffix: ${index}`);
286
+ }
287
+ if (compressedLength > 510) {
288
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} FSST suffix is too long: ${index}`);
289
+ }
290
+ current = decodeFsstValue(bytes, dictionary, cursor, compressedLength, blockPrevious, prefix, name, index);
291
+ cursor += compressedLength;
292
+ }
293
+ let value;
294
+ try {
295
+ value = textDecoder.decode(current);
296
+ }
297
+ catch (cause) {
298
+ throw new DneBinaryDatabaseFormatError(`Invalid UTF-8 in binary ${name} dictionary entry: ${index}`, { cause });
299
+ }
300
+ if (previousValue !== undefined && previousValue >= value) {
301
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary is not strictly sorted: ${index}`);
302
+ }
303
+ if (retainValues) {
304
+ values.push(value);
305
+ }
306
+ previousValue = value;
307
+ blockPrevious = current;
308
+ }
309
+ if (cursor !== blockEnd) {
310
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary block length mismatch`);
311
+ }
312
+ // Keep this assignment explicit: the first entry of the next block is
313
+ // front-coded from an empty value, while the last entry is still useful
314
+ // for the next block's diagnostic context.
315
+ }
316
+ return values;
317
+ }
318
+ function decodeFsstValue(bytes, dictionary, offset, compressedLength, previous, prefix, name, index) {
319
+ const output = new Uint8Array(255);
320
+ output.set(previous.subarray(0, prefix));
321
+ let outputLength = prefix;
322
+ let cursor = offset;
323
+ const end = offset + compressedLength;
324
+ while (cursor < end) {
325
+ const code = bytes[cursor++] ?? 0;
326
+ if (code === 255) {
327
+ if (cursor >= end || outputLength >= 255) {
328
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} FSST escape: ${index}`);
329
+ }
330
+ output[outputLength++] = bytes[cursor++] ?? 0;
331
+ continue;
332
+ }
333
+ const length = bytes[dictionary.symbolsOffset + code] ?? 0;
334
+ if (!length || length > 8 || outputLength + length > 255) {
335
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} FSST symbol: ${index}`);
336
+ }
337
+ const symbolOffset = dictionary.symbolsOffset + FSST_SYMBOL_COUNT + code * 8;
338
+ output.set(bytes.subarray(symbolOffset, symbolOffset + length), outputLength);
339
+ outputLength += length;
340
+ }
341
+ return output.slice(0, outputLength);
342
+ }
343
+ function readNeighborhoods(bytes, data, header, dictionaries) {
344
+ const region = header.sections.bairros;
345
+ if (region.length < BINARY_BAIRRO_HEADER_SIZE) {
346
+ throw new DneBinaryDatabaseFormatError('Binary neighborhood table is truncated');
347
+ }
348
+ const count = data.getUint32(region.offset, true);
349
+ const originalIdWidth = readByteWidth(data.getUint8(region.offset + 4), 'original neighborhood id');
350
+ const localidadeIdWidth = readByteWidth(data.getUint8(region.offset + 5), 'original locality id');
351
+ const recordWidth = originalIdWidth + localidadeIdWidth + dictionaries.bairro.idWidth
352
+ + dictionaries.bairroAbreviado.idWidth + dictionaries.uf.idWidth;
353
+ assertRegionLength(region, BINARY_BAIRRO_HEADER_SIZE + count * recordWidth, 'neighborhood table');
354
+ validateZeroRange(bytes, region.offset + 6, region.offset + BINARY_BAIRRO_HEADER_SIZE, 'neighborhood table reserved bytes');
355
+ const recordsOffset = region.offset + BINARY_BAIRRO_HEADER_SIZE;
356
+ let previousId = 0;
357
+ let maxId = 0;
358
+ let maxLocalidadeId = 0;
359
+ for (let index = 0; index < count; index++) {
360
+ let offset = recordsOffset + index * recordWidth;
361
+ const id = readPackedInteger(data, offset, originalIdWidth);
362
+ offset += originalIdWidth;
363
+ const localityId = readPackedInteger(data, offset, localidadeIdWidth);
364
+ offset += localidadeIdWidth;
365
+ const nameId = readPackedInteger(data, offset, dictionaries.bairro.idWidth);
366
+ offset += dictionaries.bairro.idWidth;
367
+ const abbreviationId = readPackedInteger(data, offset, dictionaries.bairroAbreviado.idWidth);
368
+ offset += dictionaries.bairroAbreviado.idWidth;
369
+ const ufId = readPackedInteger(data, offset, dictionaries.uf.idWidth);
370
+ if (id <= previousId || localityId === 0 || nameId === 0 || nameId > dictionaries.bairro.count
371
+ || abbreviationId > dictionaries.bairroAbreviado.count || ufId === 0 || ufId > dictionaries.uf.count) {
372
+ throw new DneBinaryDatabaseFormatError(`Invalid binary neighborhood record: ${index}`);
373
+ }
374
+ if (!dictionaries.bairro.values[nameId - 1]) {
375
+ throw new DneBinaryDatabaseFormatError(`Binary neighborhood has an empty name: ${index}`);
376
+ }
377
+ requireCanonicalUf(dictionaries.uf.values[ufId - 1] ?? '', index);
378
+ previousId = id;
379
+ maxId = Math.max(maxId, id);
380
+ maxLocalidadeId = Math.max(maxLocalidadeId, localityId);
381
+ }
382
+ if (originalIdWidth !== integerByteWidth(maxId) || localidadeIdWidth !== integerByteWidth(maxLocalidadeId)) {
383
+ throw new DneBinaryDatabaseFormatError('Binary neighborhood table uses a noncanonical integer width');
384
+ }
385
+ return {
386
+ count,
387
+ idWidth: integerByteWidth(count),
388
+ localidadeIdWidth,
389
+ originalIdWidth,
390
+ recordWidth,
391
+ recordsOffset,
392
+ };
393
+ }
394
+ function validateMunicipalities(data, header, dictionaries) {
395
+ const recordWidth = 3 + dictionaries.municipio.idWidth + dictionaries.uf.idWidth;
396
+ const region = header.sections.municipalities;
397
+ assertRegionLength(region, header.municipalityCount * recordWidth, 'municipalities');
398
+ const ufIds = new Uint32Array(header.municipalityCount + 1);
399
+ let previousCode = 0;
400
+ for (let index = 0; index < header.municipalityCount; index++) {
401
+ const offset = region.offset + index * recordWidth;
402
+ const code = readPackedInteger(data, offset, 3);
403
+ const municipioId = readPackedInteger(data, offset + 3, dictionaries.municipio.idWidth);
404
+ const ufId = readPackedInteger(data, offset + 3 + dictionaries.municipio.idWidth, dictionaries.uf.idWidth);
405
+ if (code < 1_000_000 || code > 9_999_999 || code <= previousCode) {
406
+ throw new DneBinaryDatabaseFormatError(`Invalid binary IBGE municipality code: ${code}`);
407
+ }
408
+ if (municipioId === 0 || municipioId > dictionaries.municipio.count || ufId === 0 || ufId > dictionaries.uf.count) {
409
+ throw new DneBinaryDatabaseFormatError(`Invalid binary municipality record: ${index}`);
410
+ }
411
+ if (!dictionaries.municipio.values[municipioId - 1]) {
412
+ throw new DneBinaryDatabaseFormatError(`Binary municipality has an empty name: ${index}`);
413
+ }
414
+ const uf = requireCanonicalUf(dictionaries.uf.values[ufId - 1] ?? '', index);
415
+ if (Math.floor(code / 100_000) !== stateCodes[uf]) {
416
+ throw new DneBinaryDatabaseFormatError(`IBGE municipality code does not match UF: ${code} (${uf}, ${stateCodes[uf]})`);
417
+ }
418
+ ufIds[index + 1] = ufId;
419
+ previousCode = code;
420
+ }
421
+ return { ufIds };
422
+ }
423
+ function validateSparseColumns(bytes, data, header, dictionaries) {
424
+ const localidadeNome = new Uint32Array(header.rowCount);
425
+ for (const [name, dictionary, values,] of [
426
+ ['complemento', dictionaries.complemento, undefined],
427
+ ['nome', dictionaries.nome, undefined],
428
+ ['localidadeNome', dictionaries.bairro, localidadeNome],
429
+ ]) {
430
+ validateSparseColumn(bytes, data, header.rowCount, header.sections[`${name}Bitmap`], header.sections[`${name}Ranks`], header.sections[`${name}Ids`], dictionary.idWidth, dictionary.count, name, values);
431
+ }
432
+ return { localidadeNome };
433
+ }
434
+ function validateSparseColumn(bytes, data, rowCount, bitmap, ranks, ids, idWidth, dictionaryCount, name, values) {
435
+ if (ids.length % idWidth !== 0) {
436
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} sparse ids have an invalid length`);
437
+ }
438
+ const blockCount = Math.ceil(rowCount / SPARSE_RANK_ROWS);
439
+ let dense = 0;
440
+ for (let block = 0; block < blockCount; block++) {
441
+ const expected = dense;
442
+ const rank = data.getUint32(ranks.offset + block * 4, true);
443
+ if (rank !== expected) {
444
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} sparse rank mismatch: ${block}`);
445
+ }
446
+ const start = block * SPARSE_RANK_ROWS;
447
+ const end = Math.min(rowCount, start + SPARSE_RANK_ROWS);
448
+ for (let row = start; row < end; row++) {
449
+ const bit = bytes[bitmap.offset + (row >>> 3)] ?? 0;
450
+ if ((bit & (1 << (row & 7))) === 0) {
451
+ continue;
452
+ }
453
+ if (dense >= ids.length / idWidth) {
454
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} sparse id index is outside the column`);
455
+ }
456
+ const value = readPackedInteger(data, ids.offset + dense * idWidth, idWidth);
457
+ if (value === 0 || value > dictionaryCount) {
458
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} sparse id: ${value}`);
459
+ }
460
+ if (values) {
461
+ values[row] = value;
462
+ }
463
+ dense++;
464
+ }
465
+ }
466
+ const finalRank = data.getUint32(ranks.offset + blockCount * 4, true);
467
+ if (finalRank !== dense || ids.length !== dense * idWidth) {
468
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} sparse column has an invalid final rank`);
469
+ }
470
+ validateBitmapPadding(bytes, bitmap, rowCount, name);
471
+ }
472
+ function validateRunSections(data, header, bairroRuns, localityRuns) {
473
+ assertRegionLength(header.sections.localidadeFlags, localityRuns.count, 'locality indicators');
474
+ validateBairroRunPadding(data, bairroRuns);
475
+ validateLocalityRunValues(data, header.sections.localidadeFlags, header, localityRuns);
476
+ }
477
+ function validateBairroRunPadding(data, runs) {
478
+ const usedLowBits = runs.runCount * runs.lowBits;
479
+ if (usedLowBits % 8 !== 0 && (data.getUint8(runs.lowOffset + Math.floor(usedLowBits / 8)) & (0xff << (usedLowBits & 7))) !== 0) {
480
+ throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood run low-bit padding');
481
+ }
482
+ }
483
+ function validateLocalityRunValues(data, flagsRegion, header, runs) {
484
+ let previousId = -1;
485
+ let previousFlags = -1;
486
+ for (let index = 0; index < runs.count; index++) {
487
+ const id = readPackedInteger(data, runs.idsOffset + index * header.municipalityIdWidth, header.municipalityIdWidth);
488
+ const flags = data.getUint8(flagsRegion.offset + index);
489
+ if (id === 0 || id > header.municipalityCount
490
+ || (flags & 3) >= LOCALIDADE_SITUACOES.length
491
+ || (flags >>> 2) >= LOCALIDADE_TIPOS.length
492
+ || (id === previousId && flags === previousFlags)) {
493
+ throw new DneBinaryDatabaseFormatError(`Invalid binary locality run: ${index}`);
494
+ }
495
+ previousId = id;
496
+ previousFlags = flags;
497
+ }
498
+ }
499
+ function validateNeighborhoodRanges(data, header, bairros) {
500
+ const { bairroFaixaOffsets: offsets, bairroFaixas: ranges } = header.sections;
501
+ assertRegionLength(offsets, (bairros.count + 1) * 4, 'neighborhood range offsets');
502
+ if (ranges.length % 8 !== 0 || data.getUint32(offsets.offset, true) !== 0) {
503
+ throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood range layout');
504
+ }
505
+ const count = ranges.length / 8;
506
+ let start = 0;
507
+ for (let index = 0; index < bairros.count; index++) {
508
+ const end = data.getUint32(offsets.offset + (index + 1) * 4, true);
509
+ if (end < start || end > count) {
510
+ throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood range offsets');
511
+ }
512
+ let previousInitial = -1;
513
+ let previousFinal = -1;
514
+ for (let row = start; row < end; row++) {
515
+ const initial = data.getUint32(ranges.offset + row * 8, true);
516
+ const final = data.getUint32(ranges.offset + row * 8 + 4, true);
517
+ if (initial > final || final > 99_999_999 || initial < previousInitial
518
+ || (initial === previousInitial && final <= previousFinal)) {
519
+ throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood CEP range');
520
+ }
521
+ previousInitial = initial;
522
+ previousFinal = final;
523
+ }
524
+ start = end;
525
+ }
526
+ if (start !== count) {
527
+ throw new DneBinaryDatabaseFormatError('Unreferenced binary neighborhood CEP ranges');
528
+ }
529
+ }
530
+ function validateCepColumns(bytes, data, header) {
531
+ const region = header.sections.cepPrefixOffsets;
532
+ const offsets = new Uint32Array(CEP_PREFIX_OFFSETS_COUNT);
533
+ let previous = 0;
534
+ for (let prefix = 0; prefix < CEP_PREFIX_OFFSETS_COUNT; prefix++) {
535
+ const value = readPackedInteger(data, region.offset + prefix * header.cepPrefixOffsetWidth, header.cepPrefixOffsetWidth);
536
+ if (value < previous || value > header.rowCount) {
537
+ throw new DneBinaryDatabaseFormatError(`Invalid binary CEP prefix boundary: ${prefix}`);
538
+ }
539
+ offsets[prefix] = value;
540
+ previous = value;
541
+ }
542
+ if (offsets[0] !== 0 || offsets[CEP_PREFIX_COUNT] !== header.rowCount) {
543
+ throw new DneBinaryDatabaseFormatError('Binary CEP prefix directory has invalid boundaries');
544
+ }
545
+ for (let prefix = 0; prefix < CEP_PREFIX_COUNT; prefix++) {
546
+ const start = offsets[prefix] ?? 0;
547
+ const end = offsets[prefix + 1] ?? 0;
548
+ let previousSuffix = -1;
549
+ for (let row = start; row < end; row++) {
550
+ const suffix = readPacked10(bytes, header.sections.cepSuffixes.offset, row);
551
+ if (suffix > 999 || suffix <= previousSuffix) {
552
+ throw new DneBinaryDatabaseFormatError(`Binary CEP suffixes are not strictly ordered: ${row}`);
553
+ }
554
+ previousSuffix = suffix;
555
+ }
556
+ }
557
+ const suffixBits = header.rowCount * CEP_SUFFIX_BITS;
558
+ if (suffixBits % 8 !== 0) {
559
+ const last = bytes[header.sections.cepSuffixes.offset + header.sections.cepSuffixes.length - 1] ?? 0;
560
+ if ((last & (0xff << (suffixBits & 7))) !== 0) {
561
+ throw new DneBinaryDatabaseFormatError('Invalid binary CEP suffix padding');
562
+ }
563
+ }
564
+ }
565
+ function validateLogradouroIds(bytes, data, header, dictionary) {
566
+ const region = header.sections.logradouroIds;
567
+ const width = integerBitWidth(dictionary.count);
568
+ assertRegionLength(region, packedBitLength(header.rowCount, width), 'logradouro ids');
569
+ for (let row = 0; row < header.rowCount; row++) {
570
+ const id = readPackedBits(data, region.offset, row, width);
571
+ if (id > dictionary.count) {
572
+ throw new DneBinaryDatabaseFormatError(`Invalid binary logradouro id: ${id}`);
573
+ }
574
+ }
575
+ const usedBits = header.rowCount * width;
576
+ if (usedBits % 8 !== 0) {
577
+ const last = bytes[region.offset + region.length - 1] ?? 0;
578
+ if ((last & (0xff << (usedBits & 7))) !== 0) {
579
+ throw new DneBinaryDatabaseFormatError('Invalid binary logradouro id padding');
580
+ }
581
+ }
582
+ }
583
+ function validateRows(data, header, dictionaries, bairros, bairroRuns, localityRuns, sparse, municipalityTable) {
584
+ const bairroRunValues = decodeBairroRuns(data, bairroRuns);
585
+ const localityRunValues = decodeLocalityRuns(data, header, localityRuns);
586
+ let bairroRun = 0;
587
+ let localityRun = 0;
588
+ for (let row = 0; row < header.rowCount; row++) {
589
+ while (bairroRun + 1 < bairroRunValues.starts.length && (bairroRunValues.starts[bairroRun + 1] ?? 0) <= row) {
590
+ bairroRun++;
591
+ }
592
+ while (localityRun + 1 < localityRunValues.starts.length && (localityRunValues.starts[localityRun + 1] ?? 0) <= row) {
593
+ localityRun++;
594
+ }
595
+ const bairroIndex = bairroRunValues.ids[bairroRun] ?? 0;
596
+ const municipalityId = localityRunValues.ids[localityRun] ?? 0;
597
+ const flags = localityRunValues.flags[localityRun] ?? 0;
598
+ const tipo = flags >>> 2;
599
+ const localidadeNomeId = sparse.localidadeNome[row] ?? 0;
600
+ if (localidadeNomeId !== 0 && (bairroIndex !== 0 || tipo === 0)) {
601
+ throw new DneBinaryDatabaseFormatError(`Invalid binary subordinate locality name: ${row}`);
602
+ }
603
+ if (bairroIndex !== 0) {
604
+ const bairroUfId = readNeighborhoodUfId(data, bairros, dictionaries, bairroIndex);
605
+ if (bairroUfId !== municipalityTable.ufIds[municipalityId]) {
606
+ throw new DneBinaryDatabaseFormatError(`Neighborhood and municipality UFs differ: ${row}`);
607
+ }
608
+ }
609
+ }
610
+ }
611
+ function readNeighborhoodUfId(data, bairros, dictionaries, index) {
612
+ const offset = bairros.recordsOffset + (index - 1) * bairros.recordWidth
613
+ + bairros.originalIdWidth + bairros.localidadeIdWidth
614
+ + dictionaries.bairro.idWidth + dictionaries.bairroAbreviado.idWidth;
615
+ return readPackedInteger(data, offset, dictionaries.uf.idWidth);
616
+ }
617
+ function decodeBairroRuns(data, runs) {
618
+ const starts = new Uint32Array(runs.runCount);
619
+ const ids = new Uint32Array(runs.runCount);
620
+ const lowBase = 2 ** runs.lowBits;
621
+ let zeros = 0;
622
+ let run = 0;
623
+ for (let bit = 0; bit < runs.highBitCount; bit++) {
624
+ const one = (data.getUint8(runs.highOffset + (bit >>> 3)) & (1 << (bit & 7))) !== 0;
625
+ if (!one) {
626
+ zeros++;
627
+ continue;
628
+ }
629
+ const low = readBits(data, runs.lowOffset, run * runs.lowBits, runs.lowBits);
630
+ starts[run] = zeros * lowBase + low;
631
+ ids[run] = readPackedInteger(data, runs.idsOffset + run * runs.idWidth, runs.idWidth);
632
+ run++;
633
+ }
634
+ return { ids, starts };
635
+ }
636
+ function decodeLocalityRuns(data, header, runs) {
637
+ const starts = new Uint32Array(runs.count);
638
+ const ids = new Uint32Array(runs.count);
639
+ const flags = new Uint8Array(runs.count);
640
+ for (let index = 0; index < runs.count; index++) {
641
+ starts[index] = data.getUint32(runs.startsOffset + index * 4, true);
642
+ ids[index] = readPackedInteger(data, runs.idsOffset + index * header.municipalityIdWidth, header.municipalityIdWidth);
643
+ flags[index] = data.getUint8(header.sections.localidadeFlags.offset + index);
644
+ }
645
+ return { flags, ids, starts };
646
+ }
647
+ function validateMetadata(bytes, region) {
648
+ const metadata = bytes.subarray(region.offset, region.offset + region.length);
649
+ if (metadata[0] === 0xef && metadata[1] === 0xbb && metadata[2] === 0xbf) {
650
+ throw new DneBinaryDatabaseFormatError('Binary database metadata must not contain a UTF-8 BOM');
651
+ }
652
+ let parsed;
653
+ try {
654
+ parsed = JSON.parse(textDecoder.decode(metadata));
655
+ }
656
+ catch (cause) {
657
+ throw new DneBinaryDatabaseFormatError('Binary database metadata is invalid JSON or UTF-8', { cause });
658
+ }
659
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
660
+ throw new DneBinaryDatabaseFormatError('Binary database metadata must be an object');
661
+ }
662
+ for (const value of Object.values(parsed)) {
663
+ if (typeof value !== 'string') {
664
+ throw new DneBinaryDatabaseFormatError('Binary database metadata values must be strings');
665
+ }
666
+ }
667
+ }
668
+ function validateRegion(fileLength, region, name) {
669
+ if (region.offset < BINARY_DATABASE_HEADER_SIZE
670
+ || region.offset > fileLength
671
+ || region.length > fileLength - region.offset) {
672
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} region is outside the payload`);
673
+ }
674
+ }
675
+ function assertRegionLength(region, expected, name) {
676
+ if (region.length !== expected) {
677
+ throw new DneBinaryDatabaseFormatError(`Binary ${name} region has an invalid length`);
678
+ }
679
+ }
680
+ function validateZeroRange(bytes, start, end, name) {
681
+ for (let index = start; index < end; index++) {
682
+ if ((bytes[index] ?? 0) !== 0) {
683
+ throw new DneBinaryDatabaseFormatError(`Invalid ${name}`);
684
+ }
685
+ }
686
+ }
687
+ function validateBitmapPadding(bytes, region, rowCount, name) {
688
+ if (rowCount % 8 !== 0) {
689
+ const last = bytes[region.offset + region.length - 1] ?? 0;
690
+ if ((last & (0xff << (rowCount & 7))) !== 0) {
691
+ throw new DneBinaryDatabaseFormatError(`Invalid binary ${name} bitmap padding`);
692
+ }
693
+ }
694
+ }
695
+ function readPacked10(bytes, offset, index) {
696
+ const bit = index * CEP_SUFFIX_BITS;
697
+ const byte = offset + Math.floor(bit / 8);
698
+ const shift = bit & 7;
699
+ const value = (bytes[byte] ?? 0) | ((bytes[byte + 1] ?? 0) << 8) | ((bytes[byte + 2] ?? 0) << 16);
700
+ return (value >>> shift) & 0x3ff;
701
+ }
702
+ function readBits(data, byteOffset, bitOffset, count) {
703
+ let value = 0;
704
+ for (let bit = 0; bit < count; bit++) {
705
+ const position = bitOffset + bit;
706
+ if (data.getUint8(byteOffset + (position >>> 3)) & (1 << (position & 7))) {
707
+ value += 2 ** bit;
708
+ }
709
+ }
710
+ return value;
711
+ }
712
+ //# sourceMappingURL=binary-db-validator.js.map