@konstit/dne 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -352
- package/dist/address-search-index.d.ts +34 -0
- package/dist/address-search-index.js +326 -0
- package/dist/address-search-index.js.map +1 -0
- package/dist/address-search-pool-errors.d.ts +15 -0
- package/dist/address-search-pool-errors.js +25 -0
- package/dist/address-search-pool-errors.js.map +1 -0
- package/dist/address-search-pool.d.ts +40 -0
- package/dist/address-search-pool.js +215 -0
- package/dist/address-search-pool.js.map +1 -0
- package/dist/address-search-rank.d.ts +5 -0
- package/dist/address-search-rank.js +89 -0
- package/dist/address-search-rank.js.map +1 -0
- package/dist/address-search-text.d.ts +6 -0
- package/dist/address-search-text.js +16 -0
- package/dist/address-search-text.js.map +1 -0
- package/dist/address-search-worker-runtime.d.ts +41 -0
- package/dist/address-search-worker-runtime.js +38 -0
- package/dist/address-search-worker-runtime.js.map +1 -0
- package/dist/address-search-worker-source.d.ts +2 -0
- package/dist/address-search-worker-source.js +5 -0
- package/dist/address-search-worker-source.js.map +1 -0
- package/dist/address-search.d.ts +31 -0
- package/dist/address-search.js +152 -0
- package/dist/address-search.js.map +1 -0
- package/dist/bairro-runs.d.ts +20 -0
- package/dist/bairro-runs.js +202 -0
- package/dist/bairro-runs.js.map +1 -0
- package/dist/binary-db-errors.d.ts +1 -1
- package/dist/binary-db-errors.js +1 -1
- package/dist/binary-db-errors.js.map +1 -1
- package/dist/binary-db-format.d.ts +10 -2
- package/dist/binary-db-format.js +9 -1
- package/dist/binary-db-format.js.map +1 -1
- package/dist/binary-db-integrity.d.ts +4 -0
- package/dist/binary-db-integrity.js +20 -0
- package/dist/binary-db-integrity.js.map +1 -0
- package/dist/binary-db-reader.d.ts +46 -28
- package/dist/binary-db-reader.js +237 -360
- package/dist/binary-db-reader.js.map +1 -1
- package/dist/binary-db-validator.d.ts +13 -0
- package/dist/binary-db-validator.js +709 -0
- package/dist/binary-db-validator.js.map +1 -0
- package/dist/chunks/dynamic-completion-wbp85hh9-cytdw6ns.js +7 -0
- package/dist/chunks/dynamic-completion-wbp85hh9-cytdw6ns.js.map +10 -0
- package/dist/chunks/index-338dn00y.js +18 -0
- package/dist/chunks/index-338dn00y.js.map +10 -0
- package/dist/expand.d.ts +8 -0
- package/dist/expand.js +1484 -0
- package/dist/expand.js.map +1 -0
- package/dist/fsst.d.ts +18 -0
- package/dist/fsst.js +117 -0
- package/dist/fsst.js.map +1 -0
- package/dist/index.js +48 -48
- package/dist/index.js.map +30 -18
- package/dist/library.d.ts +9 -2
- package/dist/library.js +9 -2
- package/dist/library.js.map +1 -1
- package/dist/locality-runs.d.ts +18 -0
- package/dist/locality-runs.js +108 -0
- package/dist/locality-runs.js.map +1 -0
- package/dist/packed-bits.d.ts +6 -0
- package/dist/packed-bits.js +41 -0
- package/dist/packed-bits.js.map +1 -0
- package/dist/settings.d.ts +0 -2
- package/dist/settings.js +0 -2
- package/dist/settings.js.map +1 -1
- package/dist/shared-file.d.ts +2 -0
- package/dist/shared-file.js +35 -0
- package/dist/shared-file.js.map +1 -0
- package/dist/sqlite-db-reader.d.ts +13 -4
- package/dist/sqlite-db-reader.js +39 -17
- package/dist/sqlite-db-reader.js.map +1 -1
- package/dist/types.d.ts +29 -4
- package/dist/types.js +73 -4
- package/dist/types.js.map +1 -1
- package/package.json +13 -10
- package/data/dne.bin +0 -0
- package/dist/chunks/dynamic-completion-403da357-4jc4w04x.js +0 -7
- package/dist/chunks/dynamic-completion-403da357-4jc4w04x.js.map +0 -10
- package/dist/chunks/index-s0h4vdva.js +0 -17
- package/dist/chunks/index-s0h4vdva.js.map +0 -10
package/dist/binary-db-reader.js
CHANGED
|
@@ -1,10 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { readBairroRunIndex, readBairroRunsLayout, } from "./bairro-runs.js";
|
|
2
3
|
import { isBairroId, } from "./bairro.js";
|
|
3
4
|
import { DneBinaryDatabaseClosedError, DneBinaryDatabaseError, DneBinaryDatabaseFormatError, DneBinaryDatabaseIOError, DneBinaryDatabaseVersionError, } from "./binary-db-errors.js";
|
|
4
|
-
import { BINARY_BAIRRO_HEADER_SIZE, BINARY_DATABASE_HEADER_SIZE, BINARY_DATABASE_MAGIC as MAGIC, BINARY_DATABASE_VERSION, BINARY_SECTION_NAMES as SECTION_NAMES,
|
|
5
|
+
import { BINARY_BAIRRO_HEADER_SIZE, BINARY_DATABASE_CHECKSUM_SIZE, BINARY_DATABASE_HEADER_SIZE, BINARY_DATABASE_MAGIC as MAGIC, BINARY_DATABASE_VERSION, BINARY_SECTION_NAMES as SECTION_NAMES, CEP_SUFFIX_BITS, createPopcountTable, DICTIONARY_BLOCK_SHIFT, DICTIONARY_CODEC_FSST, hasBinaryMagic as hasMagic, integerByteWidth, readPackedInteger as readPackedIntegerFromData, SECTION_TABLE_OFFSET, SPARSE_RANK_ROWS, SPARSE_RANK_SHIFT, } from "./binary-db-format.js";
|
|
6
|
+
import { verifyBinaryDatabaseChecksum } from "./binary-db-integrity.js";
|
|
5
7
|
import { cepToU32 } from "./cep.js";
|
|
8
|
+
import { readTrustedFsstString } from "./fsst.js";
|
|
9
|
+
import { readSharedFile } from "./shared-file.js";
|
|
10
|
+
import { findLocalityRun, readLocalityRunsLayout, } from "./locality-runs.js";
|
|
11
|
+
import { integerBitWidth, readPackedBits, } from "./packed-bits.js";
|
|
6
12
|
import { LOCALIDADE_SITUACOES, LOCALIDADE_TIPOS, } from "./schema.js";
|
|
7
|
-
import {
|
|
13
|
+
import { stateCodes, } from "./types.js";
|
|
8
14
|
/** Error classes and discriminants used by this module's public operations. */
|
|
9
15
|
export { DneBinaryDatabaseClosedError, DneBinaryDatabaseError, DneBinaryDatabaseFormatError, DneBinaryDatabaseIOError, DneBinaryDatabaseVersionError, } from "./binary-db-errors.js";
|
|
10
16
|
/** Header size and format version accepted by this reader. */
|
|
@@ -30,10 +36,11 @@ export async function isBinaryDatabase(path) {
|
|
|
30
36
|
}
|
|
31
37
|
}
|
|
32
38
|
/**
|
|
33
|
-
* Provides synchronous, read-only CEP lookups over a
|
|
39
|
+
* Provides synchronous, read-only CEP lookups over a physical or embedded DNE binary database.
|
|
34
40
|
*
|
|
35
|
-
* Construction
|
|
36
|
-
*
|
|
41
|
+
* Construction verifies the header and SHA-256 of bytes validated by a trusted generator.
|
|
42
|
+
* Queries decode directly using the invariants guaranteed at generation.
|
|
43
|
+
* The database is loaded as a snapshot at construction. Call `close()` when finished.
|
|
37
44
|
* All operational errors extend {@link DneBinaryDatabaseError}; invalid or absent CEPs return `undefined`.
|
|
38
45
|
*/
|
|
39
46
|
export class DneBinaryDatabaseReader {
|
|
@@ -42,44 +49,39 @@ export class DneBinaryDatabaseReader {
|
|
|
42
49
|
decodeScratch = new Uint8Array(0xff);
|
|
43
50
|
dictionaries;
|
|
44
51
|
bairros;
|
|
52
|
+
bairroRuns;
|
|
53
|
+
localityRuns;
|
|
54
|
+
logradouroIdBits;
|
|
45
55
|
header;
|
|
46
|
-
|
|
56
|
+
bytes;
|
|
47
57
|
metadataValue;
|
|
48
58
|
municipalityCache;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
* @param databasePath - Optional path to an existing binary database. Defaults to `data/dne.bin` inside this package.
|
|
52
|
-
* @throws {DneBinaryDatabaseIOError} If the file cannot be mapped.
|
|
53
|
-
* @throws {DneBinaryDatabaseVersionError} If the declared format version is unsupported.
|
|
54
|
-
* @throws {DneBinaryDatabaseFormatError} If the header, layout, dictionaries, or metadata are invalid.
|
|
55
|
-
*/
|
|
56
|
-
constructor(databasePath = fileURLToPath(new URL('../data/dne.bin', import.meta.url))) {
|
|
57
|
-
let mapped;
|
|
58
|
-
try {
|
|
59
|
-
mapped = Bun.mmap(databasePath, { shared: false });
|
|
60
|
-
}
|
|
61
|
-
catch (cause) {
|
|
62
|
-
throw new DneBinaryDatabaseIOError(databasePath, { cause });
|
|
63
|
-
}
|
|
59
|
+
constructor(source) {
|
|
60
|
+
const bytes = source instanceof SharedArrayBuffer ? new Uint8Array(source) : readBinaryDatabaseBytes(source);
|
|
64
61
|
try {
|
|
65
|
-
const data = new DataView(
|
|
62
|
+
const data = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
63
|
+
verifyHeader(data);
|
|
64
|
+
verifyBinaryDatabaseChecksum(bytes);
|
|
66
65
|
const header = readHeader(data);
|
|
67
|
-
validateHeader(data, header);
|
|
68
66
|
const dictionaries = readDictionaries(data, header);
|
|
69
67
|
const bairros = readNeighborhoods(data, header, dictionaries);
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
|
|
68
|
+
const bairroRuns = readBairroRunsLayout(data, header.sections.bairroIds, header.rowCount, bairros.idWidth);
|
|
69
|
+
const localityRuns = readLocalityRunsLayout(data, header.sections.municipalityIds, header.rowCount);
|
|
70
|
+
const metadata = parseMetadata(bytes, header.sections.metadata);
|
|
71
|
+
this.bytes = bytes;
|
|
73
72
|
this.data = data;
|
|
74
73
|
this.header = header;
|
|
75
74
|
this.dictionaries = dictionaries;
|
|
76
75
|
this.bairros = bairros;
|
|
77
|
-
this.
|
|
76
|
+
this.bairroRuns = bairroRuns;
|
|
77
|
+
this.localityRuns = localityRuns;
|
|
78
|
+
this.logradouroIdBits = integerBitWidth(dictionaries.logradouro.count);
|
|
79
|
+
this.metadataValue = Object.freeze(metadata);
|
|
78
80
|
this.bairroCache = Array.from({ length: dictionaries.bairro.count + 1 });
|
|
79
81
|
this.municipalityCache = Array.from({ length: header.municipalityCount + 1 });
|
|
80
82
|
}
|
|
81
83
|
catch (error) {
|
|
82
|
-
this.
|
|
84
|
+
this.bytes = null;
|
|
83
85
|
this.data = null;
|
|
84
86
|
if (error instanceof DneBinaryDatabaseError) {
|
|
85
87
|
throw error;
|
|
@@ -88,19 +90,23 @@ export class DneBinaryDatabaseReader {
|
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
/**
|
|
91
|
-
* Releases this reader's references to the
|
|
92
|
-
* Repeated calls are safe. Bun controls when the
|
|
93
|
-
*
|
|
93
|
+
* Releases this reader's references to the database bytes and decoded caches.
|
|
94
|
+
* Repeated calls are safe. Bun controls when the bytes are reclaimed by garbage collection.
|
|
95
|
+
* Metadata and row count remain available after closing.
|
|
94
96
|
*/
|
|
95
97
|
close() {
|
|
96
98
|
this.bairroCache = [];
|
|
97
99
|
this.data = null;
|
|
98
|
-
this.
|
|
100
|
+
this.bytes = null;
|
|
99
101
|
this.municipalityCache = [];
|
|
100
102
|
}
|
|
103
|
+
/** Calls close() when a using declaration leaves its scope. Repeated calls are safe. */
|
|
104
|
+
[Symbol.dispose]() {
|
|
105
|
+
this.close();
|
|
106
|
+
}
|
|
101
107
|
/**
|
|
102
108
|
* Returns the load metadata parsed when the reader was constructed.
|
|
103
|
-
* @returns The cached metadata object
|
|
109
|
+
* @returns The cached, frozen metadata object. Available after `close()`.
|
|
104
110
|
*/
|
|
105
111
|
metadata() {
|
|
106
112
|
return this.metadataValue;
|
|
@@ -110,74 +116,127 @@ export class DneBinaryDatabaseReader {
|
|
|
110
116
|
* @param cep - Eight ASCII digits or the form `NNNNN-NNN`; surrounding whitespace is not accepted.
|
|
111
117
|
* @returns An address with an eight-digit CEP, or `undefined` when the input is invalid or absent from the database.
|
|
112
118
|
* @throws {DneBinaryDatabaseClosedError} If called after `close()`, including for invalid CEP input.
|
|
113
|
-
* @throws {DneBinaryDatabaseFormatError} If the lookup encounters invalid indexes or record data.
|
|
114
119
|
*/
|
|
115
120
|
queryCep(cep) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
121
|
+
this.assertOpen();
|
|
122
|
+
const index = this.findCepIndex(cep);
|
|
123
|
+
return index === -1 ? undefined : this.readRow(index, cep.replace('-', ''));
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Resolves a municipality's name and UF by its IBGE code.
|
|
127
|
+
* @param municipioCodIbge - Seven-digit IBGE municipality code represented as an integer.
|
|
128
|
+
* @returns The municipality and UF, or undefined for an invalid or unknown code.
|
|
129
|
+
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
130
|
+
*/
|
|
131
|
+
queryMunicipality(municipioCodIbge) {
|
|
132
|
+
this.assertOpen();
|
|
133
|
+
if (!Number.isInteger(municipioCodIbge) || municipioCodIbge < 1_000_000 || municipioCodIbge > 9_999_999) {
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
const recordWidth = 3 + this.dictionaries.municipio.idWidth + this.dictionaries.uf.idWidth;
|
|
137
|
+
const offset = this.header.sections.municipalities.offset;
|
|
138
|
+
let low = 0;
|
|
139
|
+
let high = this.header.municipalityCount - 1;
|
|
140
|
+
while (low <= high) {
|
|
141
|
+
const middle = low + ((high - low) >>> 1);
|
|
142
|
+
const code = this.readPackedInteger(offset + middle * recordWidth, 3);
|
|
143
|
+
if (code < municipioCodIbge) {
|
|
144
|
+
low = middle + 1;
|
|
145
|
+
}
|
|
146
|
+
else if (code > municipioCodIbge) {
|
|
147
|
+
high = middle - 1;
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
const { municipio, uf } = this.readMunicipality(middle + 1);
|
|
151
|
+
return { municipio, uf };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Lists the IBGE municipality codes stored for a federative unit.
|
|
158
|
+
* @param uf - Canonical uppercase Brazilian state abbreviation.
|
|
159
|
+
* @returns Unique codes in ascending order, or an empty array for an invalid or absent UF.
|
|
160
|
+
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
161
|
+
*/
|
|
162
|
+
queryMunicipalityCodesByUf(uf) {
|
|
163
|
+
this.assertOpen();
|
|
164
|
+
if (!Object.hasOwn(stateCodes, uf)) {
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
const minimum = stateCodes[uf] * 100_000;
|
|
168
|
+
const maximum = minimum + 100_000;
|
|
169
|
+
const recordWidth = 3 + this.dictionaries.municipio.idWidth + this.dictionaries.uf.idWidth;
|
|
170
|
+
const offset = this.header.sections.municipalities.offset;
|
|
171
|
+
let low = 0;
|
|
172
|
+
let high = this.header.municipalityCount;
|
|
173
|
+
while (low < high) {
|
|
174
|
+
const middle = low + ((high - low) >>> 1);
|
|
175
|
+
if (this.readPackedInteger(offset + middle * recordWidth, 3) < minimum) {
|
|
176
|
+
low = middle + 1;
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
high = middle;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const codes = [];
|
|
183
|
+
for (let index = low; index < this.header.municipalityCount; index++) {
|
|
184
|
+
const code = this.readPackedInteger(offset + index * recordWidth, 3);
|
|
185
|
+
if (code >= maximum) {
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
codes.push(code);
|
|
189
|
+
}
|
|
190
|
+
return codes;
|
|
120
191
|
}
|
|
121
192
|
/**
|
|
122
193
|
* Reads a neighborhood by its original DNE identifier, independent of duplicate names.
|
|
123
194
|
* @param neighborhoodId - Positive `BAI_NU` identifier.
|
|
124
195
|
* @returns The neighborhood, or undefined for an invalid or unknown identifier.
|
|
125
196
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
126
|
-
* @throws {DneBinaryDatabaseFormatError} If an index, record, or string cannot be decoded.
|
|
127
197
|
*/
|
|
128
198
|
queryNeighborhood(neighborhoodId) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
});
|
|
199
|
+
this.assertOpen();
|
|
200
|
+
const index = this.findNeighborhoodIndex(neighborhoodId);
|
|
201
|
+
return index === 0 ? undefined : this.readNeighborhood(index);
|
|
133
202
|
}
|
|
134
203
|
/**
|
|
135
204
|
* Resolves the actual neighborhood of a CEP, preserving the distinction from districts and villages.
|
|
136
205
|
* @param cep - Eight ASCII digits or `NNNNN-NNN`.
|
|
137
206
|
* @returns The neighborhood, or undefined for an invalid, unknown, or neighborhood-free CEP.
|
|
138
207
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
139
|
-
* @throws {DneBinaryDatabaseFormatError} If the CEP or neighborhood data is invalid.
|
|
140
208
|
*/
|
|
141
209
|
queryNeighborhoodByCep(cep) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
});
|
|
210
|
+
this.assertOpen();
|
|
211
|
+
const row = this.findCepIndex(cep);
|
|
212
|
+
if (row === -1) {
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
const index = this.readNeighborhoodIndex(row);
|
|
216
|
+
return index === 0 ? undefined : this.readNeighborhood(index);
|
|
150
217
|
}
|
|
151
218
|
/**
|
|
152
219
|
* Reads a neighborhood's original inclusive CEP intervals without merging gaps.
|
|
153
220
|
* @param neighborhoodId - Positive `BAI_NU` identifier.
|
|
154
221
|
* @returns Intervals sorted by lower and upper bounds, or an empty array for an invalid, unknown, or rangeless neighborhood.
|
|
155
222
|
* @throws {DneBinaryDatabaseClosedError} If the reader is closed, including for invalid input.
|
|
156
|
-
* @throws {DneBinaryDatabaseFormatError} If the interval directory or bounds are invalid.
|
|
157
223
|
*/
|
|
158
224
|
queryNeighborhoodCepRanges(neighborhoodId) {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (initial > final || final > 99_999_999) {
|
|
175
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood CEP range');
|
|
176
|
-
}
|
|
177
|
-
result.push({ cep_inicial: String(initial).padStart(8, '0'), cep_final: String(final).padStart(8, '0') });
|
|
178
|
-
}
|
|
179
|
-
return result;
|
|
180
|
-
});
|
|
225
|
+
this.assertOpen();
|
|
226
|
+
const index = this.findNeighborhoodIndex(neighborhoodId);
|
|
227
|
+
if (index === 0) {
|
|
228
|
+
return [];
|
|
229
|
+
}
|
|
230
|
+
const { bairroFaixaOffsets: offsets, bairroFaixas: ranges } = this.header.sections;
|
|
231
|
+
const start = this.readUint32(offsets.offset + (index - 1) * 4);
|
|
232
|
+
const end = this.readUint32(offsets.offset + index * 4);
|
|
233
|
+
const result = [];
|
|
234
|
+
for (let row = start; row < end; row++) {
|
|
235
|
+
const initial = this.readUint32(ranges.offset + row * 8);
|
|
236
|
+
const final = this.readUint32(ranges.offset + row * 8 + 4);
|
|
237
|
+
result.push({ cep_inicial: String(initial).padStart(8, '0'), cep_final: String(final).padStart(8, '0') });
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
181
240
|
}
|
|
182
241
|
/**
|
|
183
242
|
* Returns the number of CEP records declared by the validated header.
|
|
@@ -186,16 +245,29 @@ export class DneBinaryDatabaseReader {
|
|
|
186
245
|
rowCount() {
|
|
187
246
|
return this.header.rowCount;
|
|
188
247
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
248
|
+
/** Returns the verified database SHA-256 used to bind optional indexes to this snapshot. */
|
|
249
|
+
checksum() {
|
|
250
|
+
this.assertOpen();
|
|
251
|
+
return Buffer.from(this.bytes.subarray(-BINARY_DATABASE_CHECKSUM_SIZE)).toString('hex');
|
|
252
|
+
}
|
|
253
|
+
/** Streams addresses in CEP order. Advancing the iterator after close() throws. */
|
|
254
|
+
*entries() {
|
|
255
|
+
this.assertOpen();
|
|
256
|
+
const region = this.header.sections.cepPrefixOffsets;
|
|
257
|
+
const width = this.header.cepPrefixOffsetWidth;
|
|
258
|
+
let prefix = 0;
|
|
259
|
+
for (let index = 0; index < this.header.rowCount; index++) {
|
|
260
|
+
while (index >= this.readPackedInteger(region.offset + (prefix + 1) * width, width)) {
|
|
261
|
+
prefix++;
|
|
197
262
|
}
|
|
198
|
-
|
|
263
|
+
const cep = String(prefix * 1_000 + this.readPacked10(index)).padStart(8, '0');
|
|
264
|
+
yield this.readRow(index, cep);
|
|
265
|
+
this.assertOpen();
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
assertOpen() {
|
|
269
|
+
if (this.data === null) {
|
|
270
|
+
throw new DneBinaryDatabaseClosedError();
|
|
199
271
|
}
|
|
200
272
|
}
|
|
201
273
|
findCepIndex(cep) {
|
|
@@ -210,9 +282,6 @@ export class DneBinaryDatabaseReader {
|
|
|
210
282
|
let low = this.readPackedInteger(prefixRegion.offset + prefix * width, width);
|
|
211
283
|
const prefixEnd = this.readPackedInteger(prefixRegion.offset + (prefix + 1) * width, width);
|
|
212
284
|
let high = prefixEnd;
|
|
213
|
-
if (low > prefixEnd || prefixEnd > this.header.rowCount) {
|
|
214
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary CEP prefix range: ${prefix}`);
|
|
215
|
-
}
|
|
216
285
|
while (low < high) {
|
|
217
286
|
const middle = low + ((high - low) >>> 1);
|
|
218
287
|
const current = this.readPacked10(middle);
|
|
@@ -251,11 +320,7 @@ export class DneBinaryDatabaseReader {
|
|
|
251
320
|
return id === neighborhoodId ? low + 1 : 0;
|
|
252
321
|
}
|
|
253
322
|
readNeighborhoodIndex(row) {
|
|
254
|
-
|
|
255
|
-
if (index > this.bairros.count) {
|
|
256
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary neighborhood index: ${index}`);
|
|
257
|
-
}
|
|
258
|
-
return index;
|
|
323
|
+
return readBairroRunIndex(this.data, this.bairroRuns, row);
|
|
259
324
|
}
|
|
260
325
|
readNeighborhood(index) {
|
|
261
326
|
const { originalIdWidth, localidadeIdWidth, recordsOffset, recordWidth } = this.bairros;
|
|
@@ -264,45 +329,34 @@ export class DneBinaryDatabaseReader {
|
|
|
264
329
|
offset += originalIdWidth;
|
|
265
330
|
const localidade_id = this.readPackedInteger(offset, localidadeIdWidth);
|
|
266
331
|
offset += localidadeIdWidth;
|
|
267
|
-
const nome = this.readRequiredDictionaryString(this.dictionaries.bairro, this.readPackedInteger(offset, this.dictionaries.bairro.idWidth)
|
|
332
|
+
const nome = this.readRequiredDictionaryString(this.dictionaries.bairro, this.readPackedInteger(offset, this.dictionaries.bairro.idWidth));
|
|
268
333
|
offset += this.dictionaries.bairro.idWidth;
|
|
269
334
|
const nome_abreviado = this.readDictionaryString(this.dictionaries.bairroAbreviado, this.readPackedInteger(offset, this.dictionaries.bairroAbreviado.idWidth));
|
|
270
335
|
offset += this.dictionaries.bairroAbreviado.idWidth;
|
|
271
|
-
const uf =
|
|
336
|
+
const uf = this.readRequiredDictionaryString(this.dictionaries.uf, this.readPackedInteger(offset, this.dictionaries.uf.idWidth));
|
|
272
337
|
return { bairro_id, localidade_id, nome, nome_abreviado, uf };
|
|
273
338
|
}
|
|
274
339
|
readRow(index, cep) {
|
|
275
340
|
const sections = this.header.sections;
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary municipality id: ${municipalityId}`);
|
|
279
|
-
}
|
|
341
|
+
const run = findLocalityRun(this.data, this.localityRuns, index);
|
|
342
|
+
const municipalityId = this.readPackedInteger(this.localityRuns.idsOffset + run * this.header.municipalityIdWidth, this.header.municipalityIdWidth);
|
|
280
343
|
const municipality = this.readMunicipality(municipalityId);
|
|
281
|
-
const flags = this.
|
|
344
|
+
const flags = this.data.getUint8(sections.localidadeFlags.offset + run);
|
|
282
345
|
const situacao = LOCALIDADE_SITUACOES[flags & 3];
|
|
283
346
|
const tipo = LOCALIDADE_TIPOS[flags >>> 2];
|
|
284
|
-
if (situacao === undefined || tipo === undefined) {
|
|
285
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary locality indicators: ${flags}`);
|
|
286
|
-
}
|
|
287
347
|
const bairroIndex = this.readNeighborhoodIndex(index);
|
|
288
348
|
const localidadeNomeId = this.readSparseId(sections.localidadeNomeBitmap, sections.localidadeNomeRanks, sections.localidadeNomeIds, index, this.dictionaries.bairro.idWidth);
|
|
289
|
-
if (localidadeNomeId !== 0 && (bairroIndex !== 0 || tipo === 'municipio')) {
|
|
290
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary subordinate locality name');
|
|
291
|
-
}
|
|
292
349
|
const bairroNameId = bairroIndex === 0
|
|
293
350
|
? localidadeNomeId
|
|
294
351
|
: this.readPackedInteger(this.bairros.recordsOffset + (bairroIndex - 1) * this.bairros.recordWidth
|
|
295
352
|
+ this.bairros.originalIdWidth + this.bairros.localidadeIdWidth, this.dictionaries.bairro.idWidth);
|
|
296
|
-
if (bairroIndex !== 0 && bairroNameId === 0) {
|
|
297
|
-
throw new DneBinaryDatabaseFormatError('Null binary neighborhood name');
|
|
298
|
-
}
|
|
299
353
|
return {
|
|
300
354
|
bairro: this.readCachedDictionaryString(this.dictionaries.bairro, bairroNameId, this.bairroCache),
|
|
301
355
|
cep,
|
|
302
356
|
complemento: this.readDictionaryString(this.dictionaries.complemento, this.readSparseId(sections.complementoBitmap, sections.complementoRanks, sections.complementoIds, index, this.dictionaries.complemento.idWidth)),
|
|
303
357
|
localidade_situacao: situacao,
|
|
304
358
|
localidade_tipo: tipo,
|
|
305
|
-
logradouro: this.readDictionaryString(this.dictionaries.logradouro, this.
|
|
359
|
+
logradouro: this.readDictionaryString(this.dictionaries.logradouro, readPackedBits(this.data, sections.logradouroIds.offset, index, this.logradouroIdBits)),
|
|
306
360
|
municipio: municipality.municipio,
|
|
307
361
|
municipio_cod_ibge: municipality.codigoIbge,
|
|
308
362
|
nome: this.readDictionaryString(this.dictionaries.nome, this.readSparseId(sections.nomeBitmap, sections.nomeRanks, sections.nomeIds, index, this.dictionaries.nome.idWidth)),
|
|
@@ -320,16 +374,16 @@ export class DneBinaryDatabaseReader {
|
|
|
320
374
|
const ufId = this.readPackedInteger(offset + 3 + this.dictionaries.municipio.idWidth, this.dictionaries.uf.idWidth);
|
|
321
375
|
const municipality = {
|
|
322
376
|
codigoIbge: this.readPackedInteger(offset, 3),
|
|
323
|
-
municipio: this.readRequiredDictionaryString(this.dictionaries.municipio, municipioId
|
|
324
|
-
uf:
|
|
377
|
+
municipio: this.readRequiredDictionaryString(this.dictionaries.municipio, municipioId),
|
|
378
|
+
uf: this.readRequiredDictionaryString(this.dictionaries.uf, ufId),
|
|
325
379
|
};
|
|
326
380
|
this.municipalityCache[id] = municipality;
|
|
327
381
|
return municipality;
|
|
328
382
|
}
|
|
329
383
|
readSparseId(bitmap, ranks, ids, rowIndex, idWidth) {
|
|
330
|
-
const
|
|
384
|
+
const bytes = this.bytes;
|
|
331
385
|
const bitmapByteIndex = rowIndex >>> 3;
|
|
332
|
-
const bitmapByte =
|
|
386
|
+
const bitmapByte = bytes[bitmap.offset + bitmapByteIndex];
|
|
333
387
|
const bitIndex = rowIndex & 7;
|
|
334
388
|
if ((bitmapByte & (1 << bitIndex)) === 0) {
|
|
335
389
|
return 0;
|
|
@@ -338,43 +392,31 @@ export class DneBinaryDatabaseReader {
|
|
|
338
392
|
let denseIndex = this.readUint32(ranks.offset + rankBlock * 4);
|
|
339
393
|
const blockByteOffset = rankBlock * (SPARSE_RANK_ROWS >>> 3);
|
|
340
394
|
for (let offset = blockByteOffset; offset < bitmapByteIndex; offset++) {
|
|
341
|
-
denseIndex += popcount[
|
|
342
|
-
}
|
|
343
|
-
denseIndex += popcount[bitmapByte & ((1 << bitIndex) - 1)] ?? 0;
|
|
344
|
-
if ((denseIndex + 1) * idWidth > ids.length) {
|
|
345
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary sparse id index: ${denseIndex}`);
|
|
395
|
+
denseIndex += popcount[bytes[bitmap.offset + offset]];
|
|
346
396
|
}
|
|
397
|
+
denseIndex += popcount[bitmapByte & ((1 << bitIndex) - 1)];
|
|
347
398
|
return this.readPackedInteger(ids.offset + denseIndex * idWidth, idWidth);
|
|
348
399
|
}
|
|
349
400
|
readDictionaryString(dictionary, id) {
|
|
350
401
|
if (id === 0) {
|
|
351
402
|
return null;
|
|
352
403
|
}
|
|
353
|
-
|
|
354
|
-
|
|
404
|
+
const bytes = this.bytes;
|
|
405
|
+
if (dictionary.codec === DICTIONARY_CODEC_FSST) {
|
|
406
|
+
return readTrustedFsstString(bytes, this.data, dictionary, id);
|
|
355
407
|
}
|
|
356
|
-
const mapped = this.requireMapped();
|
|
357
408
|
const index = id - 1;
|
|
358
409
|
const block = index >>> DICTIONARY_BLOCK_SHIFT;
|
|
359
410
|
const blockStart = block << DICTIONARY_BLOCK_SHIFT;
|
|
360
411
|
const relativeOffset = this.readUint32(dictionary.blockOffsetsOffset + block * 4);
|
|
361
|
-
if (relativeOffset > dictionary.suffixDataLength) {
|
|
362
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary string block offset: ${relativeOffset}`);
|
|
363
|
-
}
|
|
364
412
|
let cursor = dictionary.suffixDataOffset + relativeOffset;
|
|
365
413
|
const decoded = this.decodeScratch;
|
|
366
414
|
let previousLength = 0;
|
|
367
415
|
for (let currentIndex = blockStart; currentIndex <= index; currentIndex++) {
|
|
368
|
-
const length =
|
|
369
|
-
const prefixLength =
|
|
370
|
-
if ((currentIndex === blockStart && prefixLength !== 0) || prefixLength > previousLength || prefixLength > length) {
|
|
371
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary front-coded string: ${id}`);
|
|
372
|
-
}
|
|
416
|
+
const length = bytes[dictionary.lengthsOffset + currentIndex];
|
|
417
|
+
const prefixLength = bytes[dictionary.prefixesOffset + currentIndex];
|
|
373
418
|
const suffixLength = length - prefixLength;
|
|
374
|
-
|
|
375
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary string suffix: ${id}`);
|
|
376
|
-
}
|
|
377
|
-
decoded.set(mapped.subarray(cursor, cursor + suffixLength), prefixLength);
|
|
419
|
+
decoded.set(bytes.subarray(cursor, cursor + suffixLength), prefixLength);
|
|
378
420
|
cursor += suffixLength;
|
|
379
421
|
previousLength = length;
|
|
380
422
|
}
|
|
@@ -388,57 +430,48 @@ export class DneBinaryDatabaseReader {
|
|
|
388
430
|
if (cached !== undefined) {
|
|
389
431
|
return cached;
|
|
390
432
|
}
|
|
391
|
-
const value = this.
|
|
392
|
-
|
|
393
|
-
cache[id] = value;
|
|
394
|
-
}
|
|
433
|
+
const value = this.readRequiredDictionaryString(dictionary, id);
|
|
434
|
+
cache[id] = value;
|
|
395
435
|
return value;
|
|
396
436
|
}
|
|
397
|
-
readRequiredDictionaryString(dictionary, id
|
|
398
|
-
|
|
399
|
-
if (value === null) {
|
|
400
|
-
throw new DneBinaryDatabaseFormatError(`Binary database has a null required field: ${field}`);
|
|
401
|
-
}
|
|
402
|
-
return value;
|
|
437
|
+
readRequiredDictionaryString(dictionary, id) {
|
|
438
|
+
return this.readDictionaryString(dictionary, id);
|
|
403
439
|
}
|
|
404
440
|
readPacked10(index) {
|
|
405
|
-
const
|
|
441
|
+
const bytes = this.bytes;
|
|
406
442
|
const region = this.header.sections.cepSuffixes;
|
|
407
443
|
const bitOffset = index * CEP_SUFFIX_BITS;
|
|
408
444
|
const byteIndex = bitOffset >>> 3;
|
|
409
445
|
const shift = bitOffset & 7;
|
|
410
446
|
const absoluteOffset = region.offset + byteIndex;
|
|
411
|
-
const value =
|
|
412
|
-
| (
|
|
413
|
-
| (
|
|
447
|
+
const value = bytes[absoluteOffset]
|
|
448
|
+
| (bytes[absoluteOffset + 1] << 8)
|
|
449
|
+
| (bytes[absoluteOffset + 2] << 16);
|
|
414
450
|
return (value >>> shift) & 0x3ff;
|
|
415
451
|
}
|
|
416
452
|
readPackedInteger(offset, width) {
|
|
417
|
-
return readPackedIntegerFromData(this.
|
|
453
|
+
return readPackedIntegerFromData(this.data, offset, width);
|
|
418
454
|
}
|
|
419
455
|
readUint32(offset) {
|
|
420
|
-
return this.
|
|
456
|
+
return this.data.getUint32(offset, true);
|
|
421
457
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
return
|
|
458
|
+
}
|
|
459
|
+
/** @internal Reads one owned file buffer from the supplied database path. */
|
|
460
|
+
export function readBinaryDatabaseBytes(databasePath, shared = false) {
|
|
461
|
+
try {
|
|
462
|
+
return shared ? new Uint8Array(readSharedFile(databasePath)) : readFileSync(databasePath);
|
|
427
463
|
}
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
throw new DneBinaryDatabaseClosedError();
|
|
431
|
-
}
|
|
432
|
-
return this.mapped;
|
|
464
|
+
catch (cause) {
|
|
465
|
+
throw new DneBinaryDatabaseIOError(databasePath, { cause });
|
|
433
466
|
}
|
|
434
467
|
}
|
|
435
468
|
/**
|
|
436
469
|
* Opens a binary database, reads its load metadata, and closes the temporary reader.
|
|
437
470
|
* @param path - Path to an existing binary database file.
|
|
438
|
-
* @returns
|
|
439
|
-
* @throws {DneBinaryDatabaseIOError} If the file cannot be
|
|
471
|
+
* @returns Frozen metadata parsed from the file at open time.
|
|
472
|
+
* @throws {DneBinaryDatabaseIOError} If the file cannot be read.
|
|
440
473
|
* @throws {DneBinaryDatabaseVersionError} If the binary version is unsupported.
|
|
441
|
-
* @throws {DneBinaryDatabaseFormatError} If the file
|
|
474
|
+
* @throws {DneBinaryDatabaseFormatError} If the minimum header, file size, or SHA-256 checksum is invalid.
|
|
442
475
|
*/
|
|
443
476
|
export async function readBinaryDatabaseMetadata(path) {
|
|
444
477
|
const reader = new DneBinaryDatabaseReader(path);
|
|
@@ -449,8 +482,8 @@ export async function readBinaryDatabaseMetadata(path) {
|
|
|
449
482
|
reader.close();
|
|
450
483
|
}
|
|
451
484
|
}
|
|
452
|
-
function
|
|
453
|
-
if (data.byteLength < BINARY_DATABASE_HEADER_SIZE) {
|
|
485
|
+
function verifyHeader(data) {
|
|
486
|
+
if (data.byteLength < BINARY_DATABASE_HEADER_SIZE + BINARY_DATABASE_CHECKSUM_SIZE) {
|
|
454
487
|
throw new DneBinaryDatabaseFormatError('Binary database header is truncated');
|
|
455
488
|
}
|
|
456
489
|
for (let index = 0; index < MAGIC.length; index++) {
|
|
@@ -471,6 +504,11 @@ function readHeader(data) {
|
|
|
471
504
|
if (data.getUint16(24, true) !== SECTION_NAMES.length) {
|
|
472
505
|
throw new DneBinaryDatabaseFormatError('Unsupported binary database section table');
|
|
473
506
|
}
|
|
507
|
+
if (data.getUint32(16, true) !== data.byteLength) {
|
|
508
|
+
throw new DneBinaryDatabaseFormatError(`Binary database size mismatch: header=${data.getUint32(16, true)}, actual=${data.byteLength}`);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
function readHeader(data) {
|
|
474
512
|
const sections = {};
|
|
475
513
|
for (const [index, name,] of SECTION_NAMES.entries()) {
|
|
476
514
|
const offset = SECTION_TABLE_OFFSET + index * 8;
|
|
@@ -480,221 +518,60 @@ function readHeader(data) {
|
|
|
480
518
|
};
|
|
481
519
|
}
|
|
482
520
|
return {
|
|
483
|
-
cepPrefixOffsetWidth:
|
|
521
|
+
cepPrefixOffsetWidth: data.getUint8(20),
|
|
484
522
|
fileSize: data.getUint32(16, true),
|
|
485
523
|
municipalityCount: data.getUint32(28, true),
|
|
486
|
-
municipalityIdWidth:
|
|
524
|
+
municipalityIdWidth: data.getUint8(21),
|
|
487
525
|
rowCount: data.getUint32(12, true),
|
|
488
526
|
sections,
|
|
489
527
|
};
|
|
490
528
|
}
|
|
491
|
-
function validateHeader(data, header) {
|
|
492
|
-
if (header.fileSize !== data.byteLength) {
|
|
493
|
-
throw new DneBinaryDatabaseFormatError(`Binary database size mismatch: header=${header.fileSize}, actual=${data.byteLength}`);
|
|
494
|
-
}
|
|
495
|
-
if (!header.rowCount || !header.municipalityCount) {
|
|
496
|
-
throw new DneBinaryDatabaseFormatError('Binary database must contain rows and municipalities');
|
|
497
|
-
}
|
|
498
|
-
if (header.rowCount > maxPackedInteger(header.cepPrefixOffsetWidth)) {
|
|
499
|
-
throw new DneBinaryDatabaseFormatError('Binary row count does not fit the CEP prefix offset width');
|
|
500
|
-
}
|
|
501
|
-
if (header.municipalityCount > maxPackedInteger(header.municipalityIdWidth)) {
|
|
502
|
-
throw new DneBinaryDatabaseFormatError('Binary municipality count does not fit its id width');
|
|
503
|
-
}
|
|
504
|
-
let previousEnd = BINARY_DATABASE_HEADER_SIZE;
|
|
505
|
-
for (const name of SECTION_NAMES) {
|
|
506
|
-
const region = header.sections[name];
|
|
507
|
-
validateRegion(data.byteLength, region, name);
|
|
508
|
-
if (region.offset < previousEnd) {
|
|
509
|
-
throw new DneBinaryDatabaseFormatError(`Binary section is out of order or overlaps: ${name}`);
|
|
510
|
-
}
|
|
511
|
-
previousEnd = region.offset + region.length;
|
|
512
|
-
}
|
|
513
|
-
if (header.sections.cepPrefixOffsets.length !== CEP_PREFIX_OFFSETS_COUNT * header.cepPrefixOffsetWidth) {
|
|
514
|
-
throw new DneBinaryDatabaseFormatError('Binary CEP prefix directory has an invalid length');
|
|
515
|
-
}
|
|
516
|
-
if (header.sections.cepSuffixes.length !== packedBitLength(header.rowCount, CEP_SUFFIX_BITS)) {
|
|
517
|
-
throw new DneBinaryDatabaseFormatError('Binary CEP suffix column has an invalid length');
|
|
518
|
-
}
|
|
519
|
-
if (header.sections.complementoBitmap.length !== bitmapByteLength(header.rowCount)
|
|
520
|
-
|| header.sections.nomeBitmap.length !== bitmapByteLength(header.rowCount)
|
|
521
|
-
|| header.sections.localidadeNomeBitmap.length !== bitmapByteLength(header.rowCount)) {
|
|
522
|
-
throw new DneBinaryDatabaseFormatError('Binary nullable bitmap has an invalid length');
|
|
523
|
-
}
|
|
524
|
-
const expectedRankLength = (Math.ceil(header.rowCount / SPARSE_RANK_ROWS) + 1) * 4;
|
|
525
|
-
if (header.sections.complementoRanks.length !== expectedRankLength
|
|
526
|
-
|| header.sections.nomeRanks.length !== expectedRankLength
|
|
527
|
-
|| header.sections.localidadeNomeRanks.length !== expectedRankLength) {
|
|
528
|
-
throw new DneBinaryDatabaseFormatError('Binary sparse rank index has an invalid length');
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
529
|
function readDictionaries(data, header) {
|
|
532
530
|
return {
|
|
533
|
-
bairro: readDictionary(data, header.sections.bairroDictionary
|
|
534
|
-
bairroAbreviado: readDictionary(data, header.sections.bairroAbreviadoDictionary
|
|
535
|
-
complemento: readDictionary(data, header.sections.complementoDictionary
|
|
536
|
-
logradouro: readDictionary(data, header.sections.logradouroDictionary
|
|
537
|
-
municipio: readDictionary(data, header.sections.municipioDictionary
|
|
538
|
-
nome: readDictionary(data, header.sections.nomeDictionary
|
|
539
|
-
uf: readDictionary(data, header.sections.ufDictionary
|
|
531
|
+
bairro: readDictionary(data, header.sections.bairroDictionary),
|
|
532
|
+
bairroAbreviado: readDictionary(data, header.sections.bairroAbreviadoDictionary),
|
|
533
|
+
complemento: readDictionary(data, header.sections.complementoDictionary),
|
|
534
|
+
logradouro: readDictionary(data, header.sections.logradouroDictionary),
|
|
535
|
+
municipio: readDictionary(data, header.sections.municipioDictionary),
|
|
536
|
+
nome: readDictionary(data, header.sections.nomeDictionary),
|
|
537
|
+
uf: readDictionary(data, header.sections.ufDictionary),
|
|
540
538
|
};
|
|
541
539
|
}
|
|
542
|
-
function readDictionary(data, region
|
|
543
|
-
if (region.length < DICTIONARY_HEADER_SIZE) {
|
|
544
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary is truncated`);
|
|
545
|
-
}
|
|
540
|
+
function readDictionary(data, region) {
|
|
546
541
|
const count = data.getUint32(region.offset, true);
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
const suffixDataRelative = data.getUint32(region.offset + 20, true);
|
|
552
|
-
const suffixDataLength = data.getUint32(region.offset + 24, true);
|
|
553
|
-
const idWidth = readByteWidth(data.getUint8(region.offset + 28), `${name} dictionary id`);
|
|
554
|
-
if (data.getUint8(region.offset + 29) !== DICTIONARY_BLOCK_SHIFT) {
|
|
555
|
-
throw new DneBinaryDatabaseFormatError(`Unsupported binary ${name} dictionary block size`);
|
|
556
|
-
}
|
|
557
|
-
if (blockCount !== Math.ceil(count / DICTIONARY_BLOCK_SIZE)) {
|
|
558
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid block count`);
|
|
559
|
-
}
|
|
560
|
-
if (count > maxPackedInteger(idWidth)) {
|
|
561
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary does not fit its id width`);
|
|
562
|
-
}
|
|
563
|
-
if (blockOffsetsRelative !== DICTIONARY_HEADER_SIZE
|
|
564
|
-
|| lengthsRelative !== blockOffsetsRelative + blockCount * 4
|
|
565
|
-
|| prefixesRelative !== lengthsRelative + count
|
|
566
|
-
|| suffixDataRelative !== prefixesRelative + count
|
|
567
|
-
|| suffixDataRelative > region.length
|
|
568
|
-
|| suffixDataLength !== region.length - suffixDataRelative) {
|
|
569
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid layout`);
|
|
570
|
-
}
|
|
571
|
-
const dictionary = {
|
|
572
|
-
blockCount,
|
|
573
|
-
blockOffsetsOffset: region.offset + blockOffsetsRelative,
|
|
542
|
+
const prefixesOffset = region.offset + data.getUint32(region.offset + 16, true);
|
|
543
|
+
return {
|
|
544
|
+
blockCount: data.getUint32(region.offset + 4, true),
|
|
545
|
+
blockOffsetsOffset: region.offset + data.getUint32(region.offset + 8, true),
|
|
574
546
|
count,
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
547
|
+
codec: data.getUint8(region.offset + 30),
|
|
548
|
+
idWidth: data.getUint8(region.offset + 28),
|
|
549
|
+
lengthsOffset: region.offset + data.getUint32(region.offset + 12, true),
|
|
550
|
+
lengthWidth: data.getUint8(region.offset + 31) === 2 ? 2 : 1,
|
|
551
|
+
prefixesOffset,
|
|
578
552
|
region,
|
|
579
|
-
suffixDataLength,
|
|
580
|
-
suffixDataOffset: region.offset +
|
|
553
|
+
suffixDataLength: data.getUint32(region.offset + 24, true),
|
|
554
|
+
suffixDataOffset: region.offset + data.getUint32(region.offset + 20, true),
|
|
555
|
+
symbolsOffset: prefixesOffset + count,
|
|
581
556
|
};
|
|
582
|
-
if (blockCount && data.getUint32(dictionary.blockOffsetsOffset, true) !== 0) {
|
|
583
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} dictionary has an invalid first block offset`);
|
|
584
|
-
}
|
|
585
|
-
return dictionary;
|
|
586
557
|
}
|
|
587
558
|
function readNeighborhoods(data, header, dictionaries) {
|
|
588
559
|
const region = header.sections.bairros;
|
|
589
|
-
if (region.length < BINARY_BAIRRO_HEADER_SIZE) {
|
|
590
|
-
throw new DneBinaryDatabaseFormatError('Binary neighborhood table is truncated');
|
|
591
|
-
}
|
|
592
560
|
const count = data.getUint32(region.offset, true);
|
|
593
|
-
const originalIdWidth =
|
|
594
|
-
const localidadeIdWidth =
|
|
561
|
+
const originalIdWidth = data.getUint8(region.offset + 4);
|
|
562
|
+
const localidadeIdWidth = data.getUint8(region.offset + 5);
|
|
595
563
|
const recordWidth = originalIdWidth + localidadeIdWidth + dictionaries.bairro.idWidth
|
|
596
564
|
+ dictionaries.bairroAbreviado.idWidth + dictionaries.uf.idWidth;
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
offset += localidadeIdWidth;
|
|
606
|
-
const nameId = readPackedIntegerFromData(data, offset, dictionaries.bairro.idWidth);
|
|
607
|
-
offset += dictionaries.bairro.idWidth;
|
|
608
|
-
const abbreviationId = readPackedIntegerFromData(data, offset, dictionaries.bairroAbreviado.idWidth);
|
|
609
|
-
offset += dictionaries.bairroAbreviado.idWidth;
|
|
610
|
-
const ufId = readPackedIntegerFromData(data, offset, dictionaries.uf.idWidth);
|
|
611
|
-
if (id <= previousId || localityId === 0 || nameId === 0 || nameId > dictionaries.bairro.count
|
|
612
|
-
|| abbreviationId > dictionaries.bairroAbreviado.count || ufId === 0 || ufId > dictionaries.uf.count) {
|
|
613
|
-
throw new DneBinaryDatabaseFormatError(`Invalid binary neighborhood record: ${index}`);
|
|
614
|
-
}
|
|
615
|
-
previousId = id;
|
|
616
|
-
}
|
|
617
|
-
return { count, idWidth: integerByteWidth(count), originalIdWidth, localidadeIdWidth, recordsOffset, recordWidth };
|
|
618
|
-
}
|
|
619
|
-
function validateNeighborhoodRanges(data, header, bairros) {
|
|
620
|
-
const { bairroFaixaOffsets: offsets, bairroFaixas: ranges } = header.sections;
|
|
621
|
-
assertRegionLength(offsets, (bairros.count + 1) * 4, 'neighborhood range offsets');
|
|
622
|
-
if (ranges.length % 8 !== 0 || data.getUint32(offsets.offset, true) !== 0) {
|
|
623
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood range layout');
|
|
624
|
-
}
|
|
625
|
-
const count = ranges.length / 8;
|
|
626
|
-
let start = 0;
|
|
627
|
-
for (let index = 0; index < bairros.count; index++) {
|
|
628
|
-
const end = data.getUint32(offsets.offset + (index + 1) * 4, true);
|
|
629
|
-
if (end < start || end > count) {
|
|
630
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood range offsets');
|
|
631
|
-
}
|
|
632
|
-
let previousInitial = -1;
|
|
633
|
-
let previousFinal = -1;
|
|
634
|
-
for (let row = start; row < end; row++) {
|
|
635
|
-
const initial = data.getUint32(ranges.offset + row * 8, true);
|
|
636
|
-
const final = data.getUint32(ranges.offset + row * 8 + 4, true);
|
|
637
|
-
if (initial > final || final > 99_999_999 || initial < previousInitial
|
|
638
|
-
|| (initial === previousInitial && final <= previousFinal)) {
|
|
639
|
-
throw new DneBinaryDatabaseFormatError('Invalid binary neighborhood CEP range');
|
|
640
|
-
}
|
|
641
|
-
previousInitial = initial;
|
|
642
|
-
previousFinal = final;
|
|
643
|
-
}
|
|
644
|
-
start = end;
|
|
645
|
-
}
|
|
646
|
-
if (start !== count) {
|
|
647
|
-
throw new DneBinaryDatabaseFormatError('Unreferenced binary neighborhood CEP ranges');
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
function validateLayout(data, header, dictionaries, bairros) {
|
|
651
|
-
const sections = header.sections;
|
|
652
|
-
assertRegionLength(sections.localidadeFlags, header.rowCount, 'locality indicators');
|
|
653
|
-
assertRegionLength(sections.logradouroIds, header.rowCount * dictionaries.logradouro.idWidth, 'logradouro ids');
|
|
654
|
-
assertRegionLength(sections.bairroIds, header.rowCount * bairros.idWidth, 'bairro indexes');
|
|
655
|
-
assertRegionLength(sections.municipalityIds, header.rowCount * header.municipalityIdWidth, 'municipality ids');
|
|
656
|
-
const municipalityRecordWidth = 3 + dictionaries.municipio.idWidth + dictionaries.uf.idWidth;
|
|
657
|
-
assertRegionLength(sections.municipalities, header.municipalityCount * municipalityRecordWidth, 'municipalities');
|
|
658
|
-
validateSparseLayout(data, sections.complementoRanks, sections.complementoIds, dictionaries.complemento.idWidth, 'complemento');
|
|
659
|
-
validateSparseLayout(data, sections.nomeRanks, sections.nomeIds, dictionaries.nome.idWidth, 'nome');
|
|
660
|
-
validateSparseLayout(data, sections.localidadeNomeRanks, sections.localidadeNomeIds, dictionaries.bairro.idWidth, 'localidade_nome');
|
|
661
|
-
validateNeighborhoodRanges(data, header, bairros);
|
|
662
|
-
const firstPrefix = readPackedIntegerFromData(data, sections.cepPrefixOffsets.offset, header.cepPrefixOffsetWidth);
|
|
663
|
-
const lastPrefix = readPackedIntegerFromData(data, sections.cepPrefixOffsets.offset + CEP_PREFIX_COUNT * header.cepPrefixOffsetWidth, header.cepPrefixOffsetWidth);
|
|
664
|
-
if (firstPrefix !== 0 || lastPrefix !== header.rowCount) {
|
|
665
|
-
throw new DneBinaryDatabaseFormatError('Binary CEP prefix directory has invalid boundaries');
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
function validateSparseLayout(data, ranks, ids, idWidth, name) {
|
|
669
|
-
const denseCount = data.getUint32(ranks.offset + ranks.length - 4, true);
|
|
670
|
-
if (ids.length !== denseCount * idWidth) {
|
|
671
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} sparse ids have an invalid length`);
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
function parseMetadata(mapped, region) {
|
|
675
|
-
const text = textDecoder.decode(mapped.subarray(region.offset, region.offset + region.length));
|
|
676
|
-
let parsed;
|
|
677
|
-
try {
|
|
678
|
-
parsed = JSON.parse(text);
|
|
679
|
-
}
|
|
680
|
-
catch (cause) {
|
|
681
|
-
throw new DneBinaryDatabaseFormatError('Binary database metadata is invalid JSON', { cause });
|
|
682
|
-
}
|
|
683
|
-
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
684
|
-
throw new DneBinaryDatabaseFormatError('Binary database metadata must be an object');
|
|
685
|
-
}
|
|
686
|
-
return parsed;
|
|
687
|
-
}
|
|
688
|
-
function validateRegion(fileLength, region, name) {
|
|
689
|
-
if (region.offset < BINARY_DATABASE_HEADER_SIZE
|
|
690
|
-
|| region.offset > fileLength
|
|
691
|
-
|| region.length > fileLength - region.offset) {
|
|
692
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} region is outside the file`);
|
|
693
|
-
}
|
|
565
|
+
return {
|
|
566
|
+
count,
|
|
567
|
+
idWidth: integerByteWidth(count),
|
|
568
|
+
originalIdWidth,
|
|
569
|
+
localidadeIdWidth,
|
|
570
|
+
recordsOffset: region.offset + BINARY_BAIRRO_HEADER_SIZE,
|
|
571
|
+
recordWidth,
|
|
572
|
+
};
|
|
694
573
|
}
|
|
695
|
-
function
|
|
696
|
-
|
|
697
|
-
throw new DneBinaryDatabaseFormatError(`Binary ${name} region has an invalid length`);
|
|
698
|
-
}
|
|
574
|
+
function parseMetadata(bytes, region) {
|
|
575
|
+
return JSON.parse(textDecoder.decode(bytes.subarray(region.offset, region.offset + region.length)));
|
|
699
576
|
}
|
|
700
577
|
//# sourceMappingURL=binary-db-reader.js.map
|