@osmix/pbf 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +144 -0
- package/dist/blobs-to-blocks.d.ts +5 -0
- package/dist/blobs-to-blocks.js +21 -0
- package/dist/blocks-to-pbf.d.ts +16 -0
- package/dist/blocks-to-pbf.js +73 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +8 -0
- package/dist/pbf-to-blobs.d.ts +6 -0
- package/dist/pbf-to-blobs.js +48 -0
- package/dist/pbf-to-blocks.d.ts +20 -0
- package/dist/pbf-to-blocks.js +53 -0
- package/dist/proto/fileformat.d.ts +26 -0
- package/dist/proto/fileformat.js +56 -0
- package/dist/proto/osmformat.d.ts +91 -0
- package/dist/proto/osmformat.js +458 -0
- package/dist/spec.d.ts +5 -0
- package/dist/spec.js +9 -0
- package/dist/utils.d.ts +27 -0
- package/dist/utils.js +92 -0
- package/package.json +49 -0
- package/src/blobs-to-blocks.ts +28 -0
- package/src/blocks-to-pbf.ts +98 -0
- package/src/index.ts +8 -0
- package/src/pbf-to-blobs.ts +56 -0
- package/src/pbf-to-blocks.ts +77 -0
- package/src/proto/fileformat.proto +68 -0
- package/src/proto/fileformat.ts +70 -0
- package/src/proto/osmformat.proto +262 -0
- package/src/proto/osmformat.ts +488 -0
- package/src/spec.ts +10 -0
- package/src/utils.ts +90 -0
- package/test/blobs-to-blocks.test.ts +73 -0
- package/test/helpers.ts +66 -0
- package/test/pbf-to-blobs.test.ts +85 -0
- package/test/read.bench.ts +42 -0
- package/test/read.test.ts +45 -0
- package/test/streams.test.ts +92 -0
- package/test/utils.bun.test.ts +327 -0
- package/test/utils.test.ts +56 -0
- package/test/utils.ts +65 -0
- package/test/verify-pbf-reading.bun.test.ts +39 -0
- package/test/write.test.ts +86 -0
- package/tsconfig.json +9 -0
- package/vitest.config.ts +7 -0
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reads an `OsmPbfHeaderBlock` message from the provided Pbf reader.
|
|
3
|
+
*/
|
|
4
|
+
export function readHeaderBlock(pbf, end) {
|
|
5
|
+
return pbf.readFields(readHeaderBlockField, {
|
|
6
|
+
required_features: [],
|
|
7
|
+
optional_features: [],
|
|
8
|
+
}, end);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Populates header block fields based on the protobuf tag encountered.
|
|
12
|
+
*/
|
|
13
|
+
function readHeaderBlockField(tag, obj, pbf) {
|
|
14
|
+
if (tag === 1)
|
|
15
|
+
obj.bbox = readHeaderBBox(pbf, pbf.readVarint() + pbf.pos);
|
|
16
|
+
else if (tag === 4)
|
|
17
|
+
obj.required_features.push(pbf.readString());
|
|
18
|
+
else if (tag === 5)
|
|
19
|
+
obj.optional_features.push(pbf.readString());
|
|
20
|
+
else if (tag === 16)
|
|
21
|
+
obj.writingprogram = pbf.readString();
|
|
22
|
+
else if (tag === 17)
|
|
23
|
+
obj.source = pbf.readString();
|
|
24
|
+
else if (tag === 32)
|
|
25
|
+
obj.osmosis_replication_timestamp = pbf.readVarint(true);
|
|
26
|
+
else if (tag === 33) {
|
|
27
|
+
obj.osmosis_replication_sequence_number = pbf.readVarint(true);
|
|
28
|
+
}
|
|
29
|
+
else if (tag === 34)
|
|
30
|
+
obj.osmosis_replication_base_url = pbf.readString();
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Serializes an `OsmPbfHeaderBlock` message into the Pbf writer.
|
|
34
|
+
*/
|
|
35
|
+
export function writeHeaderBlock(obj, pbf) {
|
|
36
|
+
if (obj.bbox)
|
|
37
|
+
pbf.writeMessage(1, writeHeaderBBox, obj.bbox);
|
|
38
|
+
if (obj.required_features) {
|
|
39
|
+
for (const item of obj.required_features)
|
|
40
|
+
pbf.writeStringField(4, item);
|
|
41
|
+
}
|
|
42
|
+
if (obj.optional_features) {
|
|
43
|
+
for (const item of obj.optional_features)
|
|
44
|
+
pbf.writeStringField(5, item);
|
|
45
|
+
}
|
|
46
|
+
if (obj.writingprogram)
|
|
47
|
+
pbf.writeStringField(16, obj.writingprogram);
|
|
48
|
+
if (obj.source)
|
|
49
|
+
pbf.writeStringField(17, obj.source);
|
|
50
|
+
if (obj.osmosis_replication_timestamp) {
|
|
51
|
+
pbf.writeVarintField(32, obj.osmosis_replication_timestamp);
|
|
52
|
+
}
|
|
53
|
+
if (obj.osmosis_replication_sequence_number) {
|
|
54
|
+
pbf.writeVarintField(33, obj.osmosis_replication_sequence_number);
|
|
55
|
+
}
|
|
56
|
+
if (obj.osmosis_replication_base_url) {
|
|
57
|
+
pbf.writeStringField(34, obj.osmosis_replication_base_url);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Reads a header bounding box and converts nanodegrees to degrees.
|
|
62
|
+
*/
|
|
63
|
+
function readHeaderBBox(pbf, end) {
|
|
64
|
+
return pbf.readFields(readHeaderBBoxField, { left: 0, right: 0, top: 0, bottom: 0 }, end);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Populates bounding box properties while converting from nanodegrees.
|
|
68
|
+
*/
|
|
69
|
+
function readHeaderBBoxField(tag, obj, pbf) {
|
|
70
|
+
if (tag === 1)
|
|
71
|
+
obj.left = pbf.readSVarint() / 1e9;
|
|
72
|
+
else if (tag === 2)
|
|
73
|
+
obj.right = pbf.readSVarint() / 1e9;
|
|
74
|
+
else if (tag === 3)
|
|
75
|
+
obj.top = pbf.readSVarint() / 1e9;
|
|
76
|
+
else if (tag === 4)
|
|
77
|
+
obj.bottom = pbf.readSVarint() / 1e9;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Serializes a header bounding box, converting degrees back to nanodegrees.
|
|
81
|
+
*/
|
|
82
|
+
function writeHeaderBBox(obj, pbf) {
|
|
83
|
+
if (obj.left)
|
|
84
|
+
pbf.writeSVarintField(1, obj.left * 1e9);
|
|
85
|
+
if (obj.right)
|
|
86
|
+
pbf.writeSVarintField(2, obj.right * 1e9);
|
|
87
|
+
if (obj.top)
|
|
88
|
+
pbf.writeSVarintField(3, obj.top * 1e9);
|
|
89
|
+
if (obj.bottom)
|
|
90
|
+
pbf.writeSVarintField(4, obj.bottom * 1e9);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Reads a primitive block containing string tables and primitive groups.
|
|
94
|
+
*/
|
|
95
|
+
export function readPrimitiveBlock(pbf, end) {
|
|
96
|
+
return pbf.readFields(readPrimitiveBlockField, {
|
|
97
|
+
stringtable: [],
|
|
98
|
+
primitivegroup: [],
|
|
99
|
+
}, end);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Populates primitive block fields based on protobuf tags.
|
|
103
|
+
*/
|
|
104
|
+
function readPrimitiveBlockField(tag, obj, pbf) {
|
|
105
|
+
if (tag === 1) {
|
|
106
|
+
obj.stringtable = readStringTable(pbf, pbf.readVarint() + pbf.pos);
|
|
107
|
+
}
|
|
108
|
+
else if (tag === 2) {
|
|
109
|
+
obj.primitivegroup.push(readPrimitiveGroup(pbf, pbf.readVarint() + pbf.pos));
|
|
110
|
+
}
|
|
111
|
+
else if (tag === 17) {
|
|
112
|
+
obj.granularity = pbf.readVarint(true);
|
|
113
|
+
obj.granularity = !obj.granularity ? 1e7 : 1e9 / obj.granularity;
|
|
114
|
+
}
|
|
115
|
+
else if (tag === 19)
|
|
116
|
+
obj.lat_offset = pbf.readVarint(true) * 1e-9;
|
|
117
|
+
else if (tag === 20)
|
|
118
|
+
obj.lon_offset = pbf.readVarint(true) * 1e-9;
|
|
119
|
+
else if (tag === 18)
|
|
120
|
+
obj.date_granularity = pbf.readVarint(true) ?? 1000;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Serializes a primitive block including its string table and primitive groups.
|
|
124
|
+
*/
|
|
125
|
+
export function writePrimitiveBlock(obj, pbf) {
|
|
126
|
+
if (obj.stringtable)
|
|
127
|
+
pbf.writeMessage(1, writeStringTable, obj.stringtable);
|
|
128
|
+
if (obj.primitivegroup) {
|
|
129
|
+
for (const item of obj.primitivegroup) {
|
|
130
|
+
pbf.writeMessage(2, writePrimitiveGroup, item);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (obj.granularity != null && obj.granularity !== 1e7) {
|
|
134
|
+
console.error("writeGranularity", obj.granularity);
|
|
135
|
+
pbf.writeVarintField(17, 1e9 / obj.granularity);
|
|
136
|
+
}
|
|
137
|
+
if (obj.lat_offset)
|
|
138
|
+
pbf.writeVarintField(19, obj.lat_offset / 1e-9);
|
|
139
|
+
if (obj.lon_offset)
|
|
140
|
+
pbf.writeVarintField(20, obj.lon_offset / 1e-9);
|
|
141
|
+
if (obj.date_granularity != null && obj.date_granularity !== 1000) {
|
|
142
|
+
pbf.writeVarintField(18, obj.date_granularity);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Reads a primitive group with collections of primitives.
|
|
147
|
+
*/
|
|
148
|
+
function readPrimitiveGroup(pbf, end) {
|
|
149
|
+
return pbf.readFields(readPrimitiveGroupField, { nodes: [], ways: [], relations: [] }, end);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Populates primitive group collections from protobuf data.
|
|
153
|
+
*/
|
|
154
|
+
function readPrimitiveGroupField(tag, obj, pbf) {
|
|
155
|
+
if (tag === 1)
|
|
156
|
+
obj.nodes.push(readNode(pbf, pbf.readVarint() + pbf.pos));
|
|
157
|
+
else if (tag === 2) {
|
|
158
|
+
obj.dense = readDenseNodes(pbf, pbf.readVarint() + pbf.pos);
|
|
159
|
+
}
|
|
160
|
+
else if (tag === 3)
|
|
161
|
+
obj.ways.push(readWay(pbf, pbf.readVarint() + pbf.pos));
|
|
162
|
+
else if (tag === 4) {
|
|
163
|
+
obj.relations.push(readRelation(pbf, pbf.readVarint() + pbf.pos));
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Serializes a primitive group to protobuf.
|
|
168
|
+
*/
|
|
169
|
+
function writePrimitiveGroup(obj, pbf) {
|
|
170
|
+
if (obj.nodes) {
|
|
171
|
+
for (const item of obj.nodes)
|
|
172
|
+
pbf.writeMessage(1, writeNode, item);
|
|
173
|
+
}
|
|
174
|
+
if (obj.dense)
|
|
175
|
+
pbf.writeMessage(2, writeDenseNodes, obj.dense);
|
|
176
|
+
if (obj.ways) {
|
|
177
|
+
for (const item of obj.ways)
|
|
178
|
+
pbf.writeMessage(3, writeWay, item);
|
|
179
|
+
}
|
|
180
|
+
if (obj.relations) {
|
|
181
|
+
for (const item of obj.relations)
|
|
182
|
+
pbf.writeMessage(4, writeRelation, item);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Reads the shared string table for a primitive block.
|
|
187
|
+
*/
|
|
188
|
+
function readStringTable(pbf, end) {
|
|
189
|
+
return pbf.readFields(readStringTableField, [], end);
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Appends string table entries as they are encountered.
|
|
193
|
+
*/
|
|
194
|
+
function readStringTableField(tag, obj, pbf) {
|
|
195
|
+
if (tag === 1)
|
|
196
|
+
obj.push(pbf.readBytes());
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Serializes string table entries.
|
|
200
|
+
*/
|
|
201
|
+
function writeStringTable(obj, pbf) {
|
|
202
|
+
if (obj) {
|
|
203
|
+
for (const item of obj)
|
|
204
|
+
pbf.writeBytesField(1, item);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Reads metadata describing a single primitive.
|
|
209
|
+
*/
|
|
210
|
+
function readInfo(pbf, end) {
|
|
211
|
+
return pbf.readFields(readInfoField, {}, end);
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Populates primitive metadata fields.
|
|
215
|
+
*/
|
|
216
|
+
function readInfoField(tag, obj, pbf) {
|
|
217
|
+
if (tag === 1)
|
|
218
|
+
obj.version = pbf.readVarint(true);
|
|
219
|
+
else if (tag === 2)
|
|
220
|
+
obj.timestamp = pbf.readVarint(true);
|
|
221
|
+
else if (tag === 3)
|
|
222
|
+
obj.changeset = pbf.readVarint(true);
|
|
223
|
+
else if (tag === 4)
|
|
224
|
+
obj.uid = pbf.readVarint(true);
|
|
225
|
+
else if (tag === 5)
|
|
226
|
+
obj.user_sid = pbf.readVarint();
|
|
227
|
+
else if (tag === 6)
|
|
228
|
+
obj.visible = pbf.readBoolean();
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Serializes primitive metadata fields.
|
|
232
|
+
*/
|
|
233
|
+
function writeInfo(obj, pbf) {
|
|
234
|
+
if (obj.version != null && obj.version !== -1) {
|
|
235
|
+
pbf.writeVarintField(1, obj.version);
|
|
236
|
+
}
|
|
237
|
+
if (obj.timestamp)
|
|
238
|
+
pbf.writeVarintField(2, obj.timestamp);
|
|
239
|
+
if (obj.changeset)
|
|
240
|
+
pbf.writeVarintField(3, obj.changeset);
|
|
241
|
+
if (obj.uid)
|
|
242
|
+
pbf.writeVarintField(4, obj.uid);
|
|
243
|
+
if (obj.user_sid)
|
|
244
|
+
pbf.writeVarintField(5, obj.user_sid);
|
|
245
|
+
if (obj.visible)
|
|
246
|
+
pbf.writeBooleanField(6, obj.visible);
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Reads dense node metadata collections.
|
|
250
|
+
*/
|
|
251
|
+
function readDenseInfo(pbf, end) {
|
|
252
|
+
return pbf.readFields(readDenseInfoField, {
|
|
253
|
+
version: [],
|
|
254
|
+
timestamp: [],
|
|
255
|
+
changeset: [],
|
|
256
|
+
uid: [],
|
|
257
|
+
user_sid: [],
|
|
258
|
+
visible: [],
|
|
259
|
+
}, end);
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Populates dense node metadata arrays from packed fields.
|
|
263
|
+
*/
|
|
264
|
+
function readDenseInfoField(tag, obj, pbf) {
|
|
265
|
+
if (tag === 1)
|
|
266
|
+
pbf.readPackedVarint(obj.version, true);
|
|
267
|
+
else if (tag === 2)
|
|
268
|
+
pbf.readPackedSVarint(obj.timestamp);
|
|
269
|
+
else if (tag === 3)
|
|
270
|
+
pbf.readPackedSVarint(obj.changeset);
|
|
271
|
+
else if (tag === 4)
|
|
272
|
+
pbf.readPackedSVarint(obj.uid);
|
|
273
|
+
else if (tag === 5)
|
|
274
|
+
pbf.readPackedSVarint(obj.user_sid);
|
|
275
|
+
else if (tag === 6)
|
|
276
|
+
pbf.readPackedBoolean(obj.visible);
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Serializes dense node metadata arrays.
|
|
280
|
+
*/
|
|
281
|
+
function writeDenseInfo(obj, pbf) {
|
|
282
|
+
if (obj.version)
|
|
283
|
+
pbf.writePackedVarint(1, obj.version);
|
|
284
|
+
if (obj.timestamp)
|
|
285
|
+
pbf.writePackedSVarint(2, obj.timestamp);
|
|
286
|
+
if (obj.changeset)
|
|
287
|
+
pbf.writePackedSVarint(3, obj.changeset);
|
|
288
|
+
if (obj.uid)
|
|
289
|
+
pbf.writePackedSVarint(4, obj.uid);
|
|
290
|
+
if (obj.user_sid)
|
|
291
|
+
pbf.writePackedSVarint(5, obj.user_sid);
|
|
292
|
+
if (obj.visible)
|
|
293
|
+
pbf.writePackedBoolean(6, obj.visible);
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Reads a node primitive from the protobuf stream.
|
|
297
|
+
*/
|
|
298
|
+
function readNode(pbf, end) {
|
|
299
|
+
return pbf.readFields(readNodeField, { id: 0, keys: [], vals: [], lat: 0, lon: 0 }, end);
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Populates node fields based on protobuf tags.
|
|
303
|
+
*/
|
|
304
|
+
function readNodeField(tag, obj, pbf) {
|
|
305
|
+
if (tag === 1)
|
|
306
|
+
obj.id = pbf.readSVarint();
|
|
307
|
+
else if (tag === 2)
|
|
308
|
+
pbf.readPackedVarint(obj.keys);
|
|
309
|
+
else if (tag === 3)
|
|
310
|
+
pbf.readPackedVarint(obj.vals);
|
|
311
|
+
else if (tag === 4)
|
|
312
|
+
obj.info = readInfo(pbf, pbf.readVarint() + pbf.pos);
|
|
313
|
+
else if (tag === 8)
|
|
314
|
+
obj.lat = pbf.readSVarint();
|
|
315
|
+
else if (tag === 9)
|
|
316
|
+
obj.lon = pbf.readSVarint();
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Serializes a node primitive to protobuf.
|
|
320
|
+
*/
|
|
321
|
+
function writeNode(obj, pbf) {
|
|
322
|
+
if (obj.id)
|
|
323
|
+
pbf.writeSVarintField(1, obj.id);
|
|
324
|
+
if (obj.keys)
|
|
325
|
+
pbf.writePackedVarint(2, obj.keys);
|
|
326
|
+
if (obj.vals)
|
|
327
|
+
pbf.writePackedVarint(3, obj.vals);
|
|
328
|
+
if (obj.info)
|
|
329
|
+
pbf.writeMessage(4, writeInfo, obj.info);
|
|
330
|
+
if (obj.lat)
|
|
331
|
+
pbf.writeSVarintField(8, obj.lat);
|
|
332
|
+
if (obj.lon)
|
|
333
|
+
pbf.writeSVarintField(9, obj.lon);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Reads dense node collections from the protobuf stream.
|
|
337
|
+
*/
|
|
338
|
+
function readDenseNodes(pbf, end) {
|
|
339
|
+
return pbf.readFields(readDenseNodesField, { id: [], lat: [], lon: [], keys_vals: [] }, end);
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Populates dense node arrays using packed encoding.
|
|
343
|
+
*/
|
|
344
|
+
function readDenseNodesField(tag, obj, pbf) {
|
|
345
|
+
if (tag === 1)
|
|
346
|
+
pbf.readPackedSVarint(obj.id);
|
|
347
|
+
else if (tag === 5) {
|
|
348
|
+
obj.denseinfo = readDenseInfo(pbf, pbf.readVarint() + pbf.pos);
|
|
349
|
+
}
|
|
350
|
+
else if (tag === 8)
|
|
351
|
+
pbf.readPackedSVarint(obj.lat);
|
|
352
|
+
else if (tag === 9)
|
|
353
|
+
pbf.readPackedSVarint(obj.lon);
|
|
354
|
+
else if (tag === 10)
|
|
355
|
+
pbf.readPackedVarint(obj.keys_vals, true);
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Serializes dense node collections back to protobuf.
|
|
359
|
+
*/
|
|
360
|
+
function writeDenseNodes(obj, pbf) {
|
|
361
|
+
if (obj.id)
|
|
362
|
+
pbf.writePackedSVarint(1, obj.id);
|
|
363
|
+
if (obj.denseinfo)
|
|
364
|
+
pbf.writeMessage(5, writeDenseInfo, obj.denseinfo);
|
|
365
|
+
if (obj.lat)
|
|
366
|
+
pbf.writePackedSVarint(8, obj.lat);
|
|
367
|
+
if (obj.lon)
|
|
368
|
+
pbf.writePackedSVarint(9, obj.lon);
|
|
369
|
+
if (obj.keys_vals)
|
|
370
|
+
pbf.writePackedVarint(10, obj.keys_vals);
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Reads a way primitive from the protobuf stream.
|
|
374
|
+
*/
|
|
375
|
+
function readWay(pbf, end) {
|
|
376
|
+
return pbf.readFields(readWayField, { id: 0, keys: [], vals: [], refs: [], lat: [], lon: [] }, end);
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Populates way fields based on protobuf tags.
|
|
380
|
+
*/
|
|
381
|
+
function readWayField(tag, obj, pbf) {
|
|
382
|
+
if (tag === 1)
|
|
383
|
+
obj.id = pbf.readVarint(true);
|
|
384
|
+
else if (tag === 2)
|
|
385
|
+
pbf.readPackedVarint(obj.keys);
|
|
386
|
+
else if (tag === 3)
|
|
387
|
+
pbf.readPackedVarint(obj.vals);
|
|
388
|
+
else if (tag === 4)
|
|
389
|
+
obj.info = readInfo(pbf, pbf.readVarint() + pbf.pos);
|
|
390
|
+
else if (tag === 8)
|
|
391
|
+
pbf.readPackedSVarint(obj.refs);
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Serializes a way primitive to protobuf.
|
|
395
|
+
*/
|
|
396
|
+
function writeWay(obj, pbf) {
|
|
397
|
+
if (obj.id)
|
|
398
|
+
pbf.writeVarintField(1, obj.id);
|
|
399
|
+
if (obj.keys)
|
|
400
|
+
pbf.writePackedVarint(2, obj.keys);
|
|
401
|
+
if (obj.vals)
|
|
402
|
+
pbf.writePackedVarint(3, obj.vals);
|
|
403
|
+
if (obj.info)
|
|
404
|
+
pbf.writeMessage(4, writeInfo, obj.info);
|
|
405
|
+
if (obj.refs)
|
|
406
|
+
pbf.writePackedSVarint(8, obj.refs);
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Reads a relation primitive from the protobuf stream.
|
|
410
|
+
*/
|
|
411
|
+
function readRelation(pbf, end) {
|
|
412
|
+
return pbf.readFields(readRelationField, {
|
|
413
|
+
id: 0,
|
|
414
|
+
keys: [],
|
|
415
|
+
vals: [],
|
|
416
|
+
roles_sid: [],
|
|
417
|
+
memids: [],
|
|
418
|
+
types: [],
|
|
419
|
+
}, end);
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Populates relation fields based on protobuf tags.
|
|
423
|
+
*/
|
|
424
|
+
function readRelationField(tag, obj, pbf) {
|
|
425
|
+
if (tag === 1)
|
|
426
|
+
obj.id = pbf.readVarint(true);
|
|
427
|
+
else if (tag === 2)
|
|
428
|
+
pbf.readPackedVarint(obj.keys);
|
|
429
|
+
else if (tag === 3)
|
|
430
|
+
pbf.readPackedVarint(obj.vals);
|
|
431
|
+
else if (tag === 4)
|
|
432
|
+
obj.info = readInfo(pbf, pbf.readVarint() + pbf.pos);
|
|
433
|
+
else if (tag === 8)
|
|
434
|
+
pbf.readPackedVarint(obj.roles_sid, true);
|
|
435
|
+
else if (tag === 9)
|
|
436
|
+
pbf.readPackedSVarint(obj.memids);
|
|
437
|
+
else if (tag === 10)
|
|
438
|
+
pbf.readPackedVarint(obj.types);
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Serializes a relation primitive to protobuf.
|
|
442
|
+
*/
|
|
443
|
+
function writeRelation(obj, pbf) {
|
|
444
|
+
if (obj.id)
|
|
445
|
+
pbf.writeVarintField(1, obj.id);
|
|
446
|
+
if (obj.keys)
|
|
447
|
+
pbf.writePackedVarint(2, obj.keys);
|
|
448
|
+
if (obj.vals)
|
|
449
|
+
pbf.writePackedVarint(3, obj.vals);
|
|
450
|
+
if (obj.info)
|
|
451
|
+
pbf.writeMessage(4, writeInfo, obj.info);
|
|
452
|
+
if (obj.roles_sid)
|
|
453
|
+
pbf.writePackedVarint(8, obj.roles_sid);
|
|
454
|
+
if (obj.memids)
|
|
455
|
+
pbf.writePackedSVarint(9, obj.memids);
|
|
456
|
+
if (obj.types)
|
|
457
|
+
pbf.writePackedVarint(10, obj.types);
|
|
458
|
+
}
|
package/dist/spec.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare const RECOMMENDED_HEADER_SIZE_BYTES: number;
|
|
2
|
+
export declare const MAX_HEADER_SIZE_BYTES: number;
|
|
3
|
+
export declare const RECOMMENDED_BLOB_SIZE_BYTES: number;
|
|
4
|
+
export declare const MAX_BLOB_SIZE_BYTES: number;
|
|
5
|
+
export declare const MAX_ENTITIES_PER_BLOCK = 8000;
|
package/dist/spec.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Recommended and maximum header and blob sizes as defined by the OSM PBF specification
|
|
2
|
+
// Header: 32 KiB and 64 KiB
|
|
3
|
+
export const RECOMMENDED_HEADER_SIZE_BYTES = 32 * 1024;
|
|
4
|
+
export const MAX_HEADER_SIZE_BYTES = 64 * 1024;
|
|
5
|
+
// Blob: 16 MiB and 32 MiB
|
|
6
|
+
export const RECOMMENDED_BLOB_SIZE_BYTES = 16 * 1024 * 1024;
|
|
7
|
+
export const MAX_BLOB_SIZE_BYTES = 32 * 1024 * 1024;
|
|
8
|
+
// Recommended maximum number of entities per block
|
|
9
|
+
export const MAX_ENTITIES_PER_BLOCK = 8_000;
|
package/dist/utils.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export type AsyncGeneratorValue<T> = T | ReadableStream<T> | AsyncGenerator<T> | Promise<T> | Promise<ReadableStream<T>> | Promise<AsyncGenerator<T>>;
|
|
2
|
+
/**
|
|
3
|
+
* Normalizes values, streams, and iterables into a unified async generator interface.
|
|
4
|
+
*/
|
|
5
|
+
export declare function toAsyncGenerator<T>(v: AsyncGeneratorValue<T>): AsyncGenerator<T>;
|
|
6
|
+
/**
|
|
7
|
+
* Returns true when executing inside the Bun runtime.
|
|
8
|
+
*/
|
|
9
|
+
export declare function isBun(): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Decompresses binary data using runtime-native APIs.
|
|
12
|
+
* Falls back to Node zlib when running under Bun to match OSM PBF expectations.
|
|
13
|
+
*/
|
|
14
|
+
export declare function decompress(data: Uint8Array): Promise<Uint8Array>;
|
|
15
|
+
/**
|
|
16
|
+
* Compresses binary data using runtime-native APIs.
|
|
17
|
+
* Falls back to Node zlib when running under Bun to match OSM PBF expectations.
|
|
18
|
+
*/
|
|
19
|
+
export declare function compress(data: Uint8Array): Promise<Uint8Array>;
|
|
20
|
+
/**
|
|
21
|
+
* Concatenates multiple `Uint8Array` segments into a contiguous array.
|
|
22
|
+
*/
|
|
23
|
+
export declare function concatUint8(...parts: Uint8Array[]): Uint8Array;
|
|
24
|
+
/**
|
|
25
|
+
* Encodes a 32-bit big-endian unsigned integer as a four-byte buffer.
|
|
26
|
+
*/
|
|
27
|
+
export declare function uint32BE(n: number): Uint8Array;
|
package/dist/utils.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { streamToBytes } from "@osmix/shared/stream-to-bytes";
|
|
2
|
+
/**
|
|
3
|
+
* Normalizes values, streams, and iterables into a unified async generator interface.
|
|
4
|
+
*/
|
|
5
|
+
export async function* toAsyncGenerator(v) {
|
|
6
|
+
if (v instanceof Promise)
|
|
7
|
+
return toAsyncGenerator(await v);
|
|
8
|
+
if (v == null)
|
|
9
|
+
throw Error("Value is null");
|
|
10
|
+
if (v instanceof ReadableStream) {
|
|
11
|
+
const reader = v.getReader();
|
|
12
|
+
while (true) {
|
|
13
|
+
const { done, value } = await reader.read();
|
|
14
|
+
if (done)
|
|
15
|
+
break;
|
|
16
|
+
yield value;
|
|
17
|
+
}
|
|
18
|
+
reader.releaseLock();
|
|
19
|
+
}
|
|
20
|
+
else if (ArrayBuffer.isView(v) || v instanceof ArrayBuffer) {
|
|
21
|
+
// Treat ArrayBuffer and TypedArrays (like Uint8Array, Buffer) as single values
|
|
22
|
+
yield v;
|
|
23
|
+
}
|
|
24
|
+
else if (typeof v === "object" &&
|
|
25
|
+
(Symbol.asyncIterator in v || Symbol.iterator in v)) {
|
|
26
|
+
return v;
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
yield v;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Returns true when executing inside the Bun runtime.
|
|
34
|
+
*/
|
|
35
|
+
export function isBun() {
|
|
36
|
+
return "Bun" in globalThis;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Decompresses binary data using runtime-native APIs.
|
|
40
|
+
* Falls back to Node zlib when running under Bun to match OSM PBF expectations.
|
|
41
|
+
*/
|
|
42
|
+
export async function decompress(data) {
|
|
43
|
+
// Check if we're in Bun runtime - use Node.js zlib for proper OSM PBF zlib format support
|
|
44
|
+
if (isBun()) {
|
|
45
|
+
const { inflateSync } = await import("node:zlib");
|
|
46
|
+
return new Uint8Array(inflateSync(data));
|
|
47
|
+
}
|
|
48
|
+
// Fallback to standard Web API
|
|
49
|
+
const decompressedStream = new Blob([data])
|
|
50
|
+
.stream()
|
|
51
|
+
.pipeThrough(new DecompressionStream("deflate"));
|
|
52
|
+
return streamToBytes(decompressedStream);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Compresses binary data using runtime-native APIs.
|
|
56
|
+
* Falls back to Node zlib when running under Bun to match OSM PBF expectations.
|
|
57
|
+
*/
|
|
58
|
+
export async function compress(data) {
|
|
59
|
+
// Check if we're in Bun runtime - use Node.js zlib for proper OSM PBF zlib format support
|
|
60
|
+
if (isBun()) {
|
|
61
|
+
const { deflateSync } = await import("node:zlib");
|
|
62
|
+
return new Uint8Array(deflateSync(data));
|
|
63
|
+
}
|
|
64
|
+
// Fallback to standard Web API
|
|
65
|
+
const stream = new CompressionStream("deflate");
|
|
66
|
+
const compressedStream = new Blob([data]).stream().pipeThrough(stream);
|
|
67
|
+
return streamToBytes(compressedStream);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Concatenates multiple `Uint8Array` segments into a contiguous array.
|
|
71
|
+
*/
|
|
72
|
+
export function concatUint8(...parts) {
|
|
73
|
+
const total = parts.reduce((n, p) => n + p.length, 0);
|
|
74
|
+
const out = new Uint8Array(total);
|
|
75
|
+
let offset = 0;
|
|
76
|
+
for (const p of parts) {
|
|
77
|
+
out.set(p, offset);
|
|
78
|
+
offset += p.length;
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Encodes a 32-bit big-endian unsigned integer as a four-byte buffer.
|
|
84
|
+
*/
|
|
85
|
+
export function uint32BE(n) {
|
|
86
|
+
const out = new Uint8Array(4);
|
|
87
|
+
out[0] = (n >>> 24) & 0xff;
|
|
88
|
+
out[1] = (n >>> 16) & 0xff;
|
|
89
|
+
out[2] = (n >>> 8) & 0xff;
|
|
90
|
+
out[3] = n & 0xff;
|
|
91
|
+
return out;
|
|
92
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json.schemastore.org/package",
|
|
3
|
+
"name": "@osmix/pbf",
|
|
4
|
+
"description": "A low level, modern, runtime agnostic OSM PBF parser and writer written in TypeScript.",
|
|
5
|
+
"version": "0.0.1",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"main": "./src/index.ts",
|
|
8
|
+
"publishConfig": {
|
|
9
|
+
"access": "public",
|
|
10
|
+
"main": "./dist/index.js",
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"exports": {
|
|
13
|
+
".": {
|
|
14
|
+
"import": "./dist/index.js",
|
|
15
|
+
"types": "./dist/index.d.ts"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"dist",
|
|
20
|
+
"README.md",
|
|
21
|
+
"LICENSE"
|
|
22
|
+
]
|
|
23
|
+
},
|
|
24
|
+
"license": "MIT",
|
|
25
|
+
"repository": {
|
|
26
|
+
"type": "git",
|
|
27
|
+
"url": "git+https://github.com/conveyal/osmix.git"
|
|
28
|
+
},
|
|
29
|
+
"homepage": "https://github.com/conveyal/osmix#readme",
|
|
30
|
+
"bugs": {
|
|
31
|
+
"url": "https://github.com/conveyal/osmix/issues"
|
|
32
|
+
},
|
|
33
|
+
"sideEffects": false,
|
|
34
|
+
"scripts": {
|
|
35
|
+
"bench": "vitest bench",
|
|
36
|
+
"build": "tsc",
|
|
37
|
+
"test": "vitest",
|
|
38
|
+
"typecheck": "tsc --noEmit"
|
|
39
|
+
},
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"@osmix/shared": "workspace:*",
|
|
42
|
+
"@types/node": "catalog:",
|
|
43
|
+
"typescript": "catalog:",
|
|
44
|
+
"vitest": "catalog:"
|
|
45
|
+
},
|
|
46
|
+
"dependencies": {
|
|
47
|
+
"pbf": "catalog:"
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import Pbf from "pbf"
|
|
2
|
+
import { readHeaderBlock, readPrimitiveBlock } from "./proto/osmformat"
|
|
3
|
+
import { webDecompress } from "./utils"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Decompresses raw OSM PBF blobs and yields typed header and primitive blocks.
|
|
7
|
+
* Expects the first blob to contain the file header and streams the rest as data blocks.
|
|
8
|
+
*/
|
|
9
|
+
export async function* osmPbfBlobsToBlocksGenerator(
|
|
10
|
+
blobs:
|
|
11
|
+
| AsyncGenerator<Uint8Array<ArrayBuffer>>
|
|
12
|
+
| Generator<Uint8Array<ArrayBuffer>>,
|
|
13
|
+
decompress: (
|
|
14
|
+
data: Uint8Array<ArrayBuffer>,
|
|
15
|
+
) => Promise<Uint8Array<ArrayBuffer>> = webDecompress,
|
|
16
|
+
) {
|
|
17
|
+
let headerRead = false
|
|
18
|
+
for await (const blob of blobs) {
|
|
19
|
+
const decompressedBlob = await decompress(blob)
|
|
20
|
+
const pbf = new Pbf(decompressedBlob)
|
|
21
|
+
if (!headerRead) {
|
|
22
|
+
headerRead = true
|
|
23
|
+
yield readHeaderBlock(pbf)
|
|
24
|
+
} else {
|
|
25
|
+
yield readPrimitiveBlock(pbf)
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|