@osmix/core 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +54 -7
- package/dist/contracts.d.ts +2 -1
- package/dist/contracts.d.ts.map +1 -1
- package/dist/entities.d.ts +21 -1
- package/dist/entities.d.ts.map +1 -1
- package/dist/entities.js +52 -3
- package/dist/entities.js.map +1 -1
- package/dist/ids.d.ts +18 -5
- package/dist/ids.d.ts.map +1 -1
- package/dist/ids.js +68 -18
- package/dist/ids.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/indirect-kd-index.d.ts +28 -0
- package/dist/indirect-kd-index.d.ts.map +1 -0
- package/dist/indirect-kd-index.js +163 -0
- package/dist/indirect-kd-index.js.map +1 -0
- package/dist/limits.d.ts +30 -0
- package/dist/limits.d.ts.map +1 -0
- package/dist/limits.js +51 -0
- package/dist/limits.js.map +1 -0
- package/dist/nodes.d.ts +28 -13
- package/dist/nodes.d.ts.map +1 -1
- package/dist/nodes.js +181 -51
- package/dist/nodes.js.map +1 -1
- package/dist/osm.d.ts +59 -3
- package/dist/osm.d.ts.map +1 -1
- package/dist/osm.js +31 -2
- package/dist/osm.js.map +1 -1
- package/dist/relations.d.ts +10 -4
- package/dist/relations.d.ts.map +1 -1
- package/dist/relations.js +66 -28
- package/dist/relations.js.map +1 -1
- package/dist/renumber.d.ts +23 -0
- package/dist/renumber.d.ts.map +1 -0
- package/dist/renumber.js +64 -0
- package/dist/renumber.js.map +1 -0
- package/dist/stringtable.d.ts +2 -1
- package/dist/stringtable.d.ts.map +1 -1
- package/dist/stringtable.js +5 -1
- package/dist/stringtable.js.map +1 -1
- package/dist/tags.d.ts +44 -11
- package/dist/tags.d.ts.map +1 -1
- package/dist/tags.js +153 -27
- package/dist/tags.js.map +1 -1
- package/dist/typed-arrays.d.ts +63 -48
- package/dist/typed-arrays.d.ts.map +1 -1
- package/dist/typed-arrays.js +184 -107
- package/dist/typed-arrays.js.map +1 -1
- package/dist/ways.d.ts +50 -2
- package/dist/ways.d.ts.map +1 -1
- package/dist/ways.js +212 -35
- package/dist/ways.js.map +1 -1
- package/package.json +11 -12
- package/src/contracts.ts +2 -1
- package/src/entities.ts +67 -4
- package/src/ids.ts +72 -22
- package/src/index.ts +13 -7
- package/src/indirect-kd-index.ts +212 -0
- package/src/limits.ts +73 -0
- package/src/nodes.ts +228 -54
- package/src/osm.ts +90 -4
- package/src/relations.ts +80 -30
- package/src/renumber.ts +80 -0
- package/src/stringtable.ts +15 -1
- package/src/tags.ts +173 -31
- package/src/typed-arrays.ts +225 -113
- package/src/ways.ts +233 -38
- package/test/entity-index-errors.test.ts +41 -0
- package/test/ids.bench.ts +11 -7
- package/test/ids.test.ts +38 -0
- package/test/limits.test.ts +83 -0
- package/test/no-shared-array-buffer.test.ts +41 -0
- package/test/nodes-spatial-index.test.ts +261 -0
- package/test/osm-bbox.test.ts +30 -0
- package/test/relations.bench.ts +9 -4
- package/test/relations.test.ts +52 -29
- package/test/renumber.test.ts +54 -0
- package/test/spatial-index-rebuild.test.ts +33 -0
- package/test/storage-compaction.test.ts +269 -0
- package/test/tags-search.test.ts +13 -0
- package/test/typed-arrays.test.ts +178 -59
- package/src/types/geokdbush.d.ts +0 -32
- package/src/types/kdbush.d.ts +0 -47
package/src/relations.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { buildRelationRings } from "@osmix/geo/relation-multipolygon";
|
|
2
2
|
import type { OsmPbfRelation } from "@osmix/pbf";
|
|
3
3
|
import { assertValue } from "@osmix/shared/assert";
|
|
4
|
+
import { BitSet } from "@osmix/shared/bit-set";
|
|
4
5
|
import type { ContentHasher } from "@osmix/shared/content-hasher";
|
|
5
6
|
import type {
|
|
6
7
|
GeoBbox2D,
|
|
@@ -23,8 +24,9 @@ import { bboxFromLonLats } from "@osmix/types/utils";
|
|
|
23
24
|
import Flatbush from "flatbush";
|
|
24
25
|
import { around as geoAround } from "geoflatbush";
|
|
25
26
|
|
|
26
|
-
import { Entities, type EntitiesTransferables } from "./entities.ts";
|
|
27
|
+
import { Entities, type EntitiesTransferables, isValidSpatialBbox } from "./entities.ts";
|
|
27
28
|
import { type IdOrIndex, Ids } from "./ids.ts";
|
|
29
|
+
import { MAX_RELATION_MEMBERS, MAX_UINT32_OFFSET, assertCapacity } from "./limits.ts";
|
|
28
30
|
import type { Nodes } from "./nodes.ts";
|
|
29
31
|
import type StringTable from "./stringtable.ts";
|
|
30
32
|
import { Tags } from "./tags.ts";
|
|
@@ -55,7 +57,7 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
55
57
|
private stringTable: StringTable;
|
|
56
58
|
|
|
57
59
|
private memberStart: RTA<Uint32Array>;
|
|
58
|
-
private memberCount: RTA<Uint16Array>; // Maximum 65,535
|
|
60
|
+
private memberCount: RTA<Uint16Array>; // Maximum MAX_RELATION_MEMBERS (65,535) per relation
|
|
59
61
|
|
|
60
62
|
// Store the ID of the member because relations have other relations as members.
|
|
61
63
|
private memberRefs: RTA<Float64Array>;
|
|
@@ -74,8 +76,9 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
74
76
|
private nodes: Nodes;
|
|
75
77
|
private ways: Ways;
|
|
76
78
|
|
|
77
|
-
// Lazily computed relation way membership. Invalidated whenever relations
|
|
78
|
-
|
|
79
|
+
// Lazily computed relation way membership keyed by way index. Invalidated whenever relations
|
|
80
|
+
// change, and rebuilt if the way count no longer matches.
|
|
81
|
+
private wayMemberCache: BitSet | null = null;
|
|
79
82
|
|
|
80
83
|
/**
|
|
81
84
|
* Create a new Relations index.
|
|
@@ -114,11 +117,28 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
114
117
|
this.stringTable = stringTable;
|
|
115
118
|
}
|
|
116
119
|
|
|
120
|
+
/** Reject a relation whose member count would wrap `memberCount` or `memberStart`. */
|
|
121
|
+
private assertMemberCapacity(relationId: number, memberCount: number) {
|
|
122
|
+
assertCapacity(
|
|
123
|
+
"relation-members",
|
|
124
|
+
memberCount,
|
|
125
|
+
MAX_RELATION_MEMBERS,
|
|
126
|
+
() => `Relation ${relationId}`,
|
|
127
|
+
);
|
|
128
|
+
assertCapacity(
|
|
129
|
+
"total-relation-members",
|
|
130
|
+
this.memberRefs.length + memberCount,
|
|
131
|
+
MAX_UINT32_OFFSET,
|
|
132
|
+
() => `The dataset (at relation ${relationId})`,
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
|
|
117
136
|
/**
|
|
118
137
|
* Add a single relation to the index.
|
|
119
138
|
*/
|
|
120
139
|
addRelation(relation: OsmRelation) {
|
|
121
|
-
this.
|
|
140
|
+
this.wayMemberCache = null;
|
|
141
|
+
this.assertMemberCapacity(relation.id, relation.members.length);
|
|
122
142
|
const relationIndex = this.addEntity(relation.id, relation.tags ?? {});
|
|
123
143
|
this.memberStart.push(this.memberRefs.length);
|
|
124
144
|
this.memberCount.push(relation.members.length);
|
|
@@ -138,7 +158,7 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
138
158
|
blockStringIndexMap: Uint32Array,
|
|
139
159
|
filter?: (relation: OsmRelation) => OsmRelation | null,
|
|
140
160
|
): number {
|
|
141
|
-
this.
|
|
161
|
+
this.wayMemberCache = null;
|
|
142
162
|
const blockToStringTable = (k: number) => {
|
|
143
163
|
const index = blockStringIndexMap[k];
|
|
144
164
|
if (index === undefined) throw Error("Tag key not found");
|
|
@@ -190,6 +210,7 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
190
210
|
})
|
|
191
211
|
: null;
|
|
192
212
|
if (filter && filteredRelation === null) continue;
|
|
213
|
+
this.assertMemberCapacity(relation.id, filteredRelation?.members.length ?? memberRefs.length);
|
|
193
214
|
added++;
|
|
194
215
|
|
|
195
216
|
this.addEntity(relation.id, tagKeys, tagValues);
|
|
@@ -221,11 +242,17 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
221
242
|
* Build the spatial index for relations.
|
|
222
243
|
* Handles nested relations by resolving all descendant nodes and ways.
|
|
223
244
|
* If bbox data already exists (e.g., loaded from storage), reuses it.
|
|
245
|
+
* Returns the existing index when it is already built.
|
|
224
246
|
*/
|
|
225
247
|
buildSpatialIndex() {
|
|
226
248
|
if (!this.nodes.isReady()) throw Error("Node index is not ready.");
|
|
227
249
|
if (!this.ways.isReady()) throw Error("Way index is not ready.");
|
|
228
|
-
if (this.
|
|
250
|
+
if (this.spatialIndexBuilt) return this.spatialIndex;
|
|
251
|
+
if (this.size === 0) {
|
|
252
|
+
this.spatialIndex = new Flatbush(1, 128, Float64Array, BufferConstructor);
|
|
253
|
+
this.spatialIndexBuilt = true;
|
|
254
|
+
return this.spatialIndex;
|
|
255
|
+
}
|
|
229
256
|
console.time("RelationIndex.buildSpatialIndex");
|
|
230
257
|
|
|
231
258
|
this.spatialIndex = new Flatbush(this.size, 128, Float64Array, BufferConstructor);
|
|
@@ -300,11 +327,12 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
300
327
|
if (ll) lls.push(ll);
|
|
301
328
|
}
|
|
302
329
|
|
|
303
|
-
// Collect coordinates from resolved ways
|
|
330
|
+
// Collect coordinates from resolved ways. Member ways may reference nodes
|
|
331
|
+
// outside a referentially incomplete extract, so skip unresolvable refs.
|
|
304
332
|
for (const wayId of resolved.ways) {
|
|
305
333
|
const wayIndex = this.ways.ids.getIndexFromId(wayId);
|
|
306
334
|
if (wayIndex === -1) continue;
|
|
307
|
-
const wayPositions = this.ways.
|
|
335
|
+
const wayPositions = this.ways.getResolvedCoordinates(wayIndex);
|
|
308
336
|
lls.push(...wayPositions);
|
|
309
337
|
}
|
|
310
338
|
|
|
@@ -372,30 +400,35 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
372
400
|
}
|
|
373
401
|
|
|
374
402
|
/**
|
|
375
|
-
*
|
|
403
|
+
* Check whether a way is a member of any relation, including through nested relations.
|
|
376
404
|
* Used to exclude these ways from individual rendering.
|
|
405
|
+
*
|
|
406
|
+
* Membership is computed once and cached as one bit per way index. Requires the way ID index.
|
|
377
407
|
*/
|
|
378
|
-
|
|
379
|
-
|
|
408
|
+
isWayMember(wayIndex: number): boolean {
|
|
409
|
+
return this.getWayMembers().has(wayIndex);
|
|
410
|
+
}
|
|
380
411
|
|
|
381
|
-
|
|
412
|
+
private getWayMembers(): BitSet {
|
|
413
|
+
if (this.wayMemberCache !== null && this.wayMemberCache.size === this.ways.size) {
|
|
414
|
+
return this.wayMemberCache;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const members = new BitSet(this.ways.size);
|
|
418
|
+
const getRelation = (relId: number) => {
|
|
419
|
+
const relIndex = this.ids.getIndexFromId(relId);
|
|
420
|
+
if (relIndex === -1) return null;
|
|
421
|
+
return this.getByIndex(relIndex);
|
|
422
|
+
};
|
|
382
423
|
for (let i = 0; i < this.size; i++) {
|
|
383
|
-
const
|
|
384
|
-
const resolved = resolveRelationMembers(
|
|
385
|
-
relation,
|
|
386
|
-
(relId) => {
|
|
387
|
-
const relIndex = this.ids.getIndexFromId(relId);
|
|
388
|
-
if (relIndex === -1) return null;
|
|
389
|
-
return this.getByIndex(relIndex);
|
|
390
|
-
},
|
|
391
|
-
10, // max depth
|
|
392
|
-
);
|
|
424
|
+
const resolved = resolveRelationMembers(this.getByIndex(i), getRelation, 10);
|
|
393
425
|
for (const wayId of resolved.ways) {
|
|
394
|
-
|
|
426
|
+
const wayIndex = this.ways.ids.getIndexFromId(wayId);
|
|
427
|
+
if (wayIndex !== -1) members.add(wayIndex);
|
|
395
428
|
}
|
|
396
429
|
}
|
|
397
|
-
this.
|
|
398
|
-
return
|
|
430
|
+
this.wayMemberCache = members;
|
|
431
|
+
return members;
|
|
399
432
|
}
|
|
400
433
|
|
|
401
434
|
/**
|
|
@@ -457,7 +490,13 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
457
490
|
*/
|
|
458
491
|
intersects(bbox: GeoBbox2D, filterFn?: (index: number) => boolean): number[] {
|
|
459
492
|
if (this.size === 0) return [];
|
|
460
|
-
|
|
493
|
+
// A relation with no resolvable member geometry stores an inverted bbox.
|
|
494
|
+
// Flatbush's contained-node fast path skips per-leaf intersection tests,
|
|
495
|
+
// so filter inverted boxes out explicitly.
|
|
496
|
+
return this.spatialIndex.search(bbox[0], bbox[1], bbox[2], bbox[3], (index, x0, y0, x1, y1) => {
|
|
497
|
+
if (!isValidSpatialBbox(x0, y0, x1, y1)) return false;
|
|
498
|
+
return filterFn ? filterFn(index) : true;
|
|
499
|
+
});
|
|
461
500
|
}
|
|
462
501
|
|
|
463
502
|
/**
|
|
@@ -471,7 +510,15 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
471
510
|
neighbors(lon: number, lat: number, maxResults?: number, maxDistanceKm?: number): number[] {
|
|
472
511
|
if (this.size === 0) return [];
|
|
473
512
|
// Use geoflatbush for proper geographic distance calculations
|
|
474
|
-
return geoAround(this.spatialIndex, lon, lat, maxResults, maxDistanceKm)
|
|
513
|
+
return geoAround(this.spatialIndex, lon, lat, maxResults, maxDistanceKm, (index) => {
|
|
514
|
+
const offset = index * 4;
|
|
515
|
+
return isValidSpatialBbox(
|
|
516
|
+
this.bbox.at(offset),
|
|
517
|
+
this.bbox.at(offset + 1),
|
|
518
|
+
this.bbox.at(offset + 2),
|
|
519
|
+
this.bbox.at(offset + 3),
|
|
520
|
+
);
|
|
521
|
+
});
|
|
475
522
|
}
|
|
476
523
|
|
|
477
524
|
/**
|
|
@@ -498,7 +545,7 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
498
545
|
/**
|
|
499
546
|
* Get the approximate memory requirements for a given number of relations in bytes.
|
|
500
547
|
*/
|
|
501
|
-
static getBytesRequired(count: number) {
|
|
548
|
+
static getBytesRequired(count: number, memberCount = 0, tagCount = 0) {
|
|
502
549
|
if (count === 0) return 0;
|
|
503
550
|
// Approximate members per relation
|
|
504
551
|
let numNodes = count;
|
|
@@ -513,9 +560,12 @@ export class Relations extends Entities<OsmRelation> {
|
|
|
513
560
|
|
|
514
561
|
return (
|
|
515
562
|
Ids.getBytesRequired(count) +
|
|
516
|
-
Tags.getBytesRequired(count) +
|
|
563
|
+
Tags.getBytesRequired(count, count, tagCount) +
|
|
517
564
|
count * Uint32Array.BYTES_PER_ELEMENT + // memberStart
|
|
518
565
|
count * Uint16Array.BYTES_PER_ELEMENT + // memberCount
|
|
566
|
+
memberCount * Float64Array.BYTES_PER_ELEMENT + // memberRefs
|
|
567
|
+
memberCount * Uint8Array.BYTES_PER_ELEMENT + // memberTypes
|
|
568
|
+
memberCount * Uint32Array.BYTES_PER_ELEMENT + // memberRoles
|
|
519
569
|
count * 4 * Float64Array.BYTES_PER_ELEMENT + // bbox
|
|
520
570
|
spatialIndexBytes
|
|
521
571
|
);
|
package/src/renumber.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { Osm } from "./osm.ts";
|
|
2
|
+
|
|
3
|
+
/** Old ID → new ID for each renumbered entity, by type. JSON- and structured-clone-safe. */
|
|
4
|
+
export interface OsmIdMap {
|
|
5
|
+
nodes: Record<number, number>;
|
|
6
|
+
ways: Record<number, number>;
|
|
7
|
+
relations: Record<number, number>;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** New positive IDs for one type's negative IDs, after its highest positive ID, in −1, −2, … order. */
|
|
11
|
+
function positiveIds(ids: { size: number; at(index: number): number }): Map<number, number> {
|
|
12
|
+
let max = 0;
|
|
13
|
+
const negative: number[] = [];
|
|
14
|
+
for (let index = 0; index < ids.size; index++) {
|
|
15
|
+
const id = ids.at(index);
|
|
16
|
+
if (id < 0) negative.push(id);
|
|
17
|
+
else max = Math.max(max, id);
|
|
18
|
+
}
|
|
19
|
+
negative.sort((a, b) => b - a);
|
|
20
|
+
return new Map(negative.map((id, index) => [id, max + index + 1]));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* A copy of `osm` whose negative node, way and relation IDs are replaced by positive ones, for
|
|
25
|
+
* tools that reject negative IDs. In OSM files a negative ID conventionally marks a new, not yet
|
|
26
|
+
* uploaded entity, so only do this when the consumer needs it. Each type gets IDs after its own
|
|
27
|
+
* highest positive ID, in −1, −2, … order, so the result is deterministic. Way refs and relation
|
|
28
|
+
* members follow. `osm` is unchanged; the copy has its indexes built.
|
|
29
|
+
*/
|
|
30
|
+
export function renumberNegativeIds(osm: Osm, options: { id?: string } = {}) {
|
|
31
|
+
const nodeIds = positiveIds(osm.nodes.ids);
|
|
32
|
+
const wayIds = positiveIds(osm.ways.ids);
|
|
33
|
+
const relationIds = positiveIds(osm.relations.ids);
|
|
34
|
+
const mapped = { node: nodeIds, way: wayIds, relation: relationIds } as const;
|
|
35
|
+
const renumbered = new Osm({ id: options.id ?? osm.id, header: osm.header });
|
|
36
|
+
for (const node of osm.nodes.sorted()) {
|
|
37
|
+
renumbered.nodes.addNode({ ...node, id: nodeIds.get(node.id) ?? node.id });
|
|
38
|
+
}
|
|
39
|
+
for (const way of osm.ways.sorted()) {
|
|
40
|
+
renumbered.ways.addWay({
|
|
41
|
+
...way,
|
|
42
|
+
id: wayIds.get(way.id) ?? way.id,
|
|
43
|
+
refs: way.refs.map((ref) => nodeIds.get(ref) ?? ref),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
for (const relation of osm.relations.sorted()) {
|
|
47
|
+
renumbered.relations.addRelation({
|
|
48
|
+
...relation,
|
|
49
|
+
id: relationIds.get(relation.id) ?? relation.id,
|
|
50
|
+
members: relation.members.map((member) => ({
|
|
51
|
+
...member,
|
|
52
|
+
ref: mapped[member.type].get(member.ref) ?? member.ref,
|
|
53
|
+
})),
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
renumbered.buildIndexes();
|
|
57
|
+
renumbered.buildSpatialIndexes();
|
|
58
|
+
return { osm: renumbered, idMap: toIdMap(nodeIds, wayIds, relationIds) };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function toIdMap(
|
|
62
|
+
nodes: Map<number, number>,
|
|
63
|
+
ways: Map<number, number>,
|
|
64
|
+
relations: Map<number, number>,
|
|
65
|
+
): OsmIdMap {
|
|
66
|
+
return {
|
|
67
|
+
nodes: Object.fromEntries(nodes),
|
|
68
|
+
ways: Object.fromEntries(ways),
|
|
69
|
+
relations: Object.fromEntries(relations),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** The ID map `renumberNegativeIds(osm)` would apply, without copying the data. */
|
|
74
|
+
export function negativeIdMap(osm: Osm): OsmIdMap {
|
|
75
|
+
return toIdMap(
|
|
76
|
+
positiveIds(osm.nodes.ids),
|
|
77
|
+
positiveIds(osm.ways.ids),
|
|
78
|
+
positiveIds(osm.relations.ids),
|
|
79
|
+
);
|
|
80
|
+
}
|
package/src/stringtable.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import type { OsmPbfStringTable } from "@osmix/pbf";
|
|
11
11
|
import type { ContentHasher } from "@osmix/shared/content-hasher";
|
|
12
12
|
|
|
13
|
+
import { MAX_STRING_BYTES, MAX_UINT32_OFFSET, assertCapacity } from "./limits.ts";
|
|
13
14
|
import { type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays.ts";
|
|
14
15
|
|
|
15
16
|
/**
|
|
@@ -27,7 +28,8 @@ export interface StringTableTransferables<T extends BufferType = BufferType> {
|
|
|
27
28
|
/**
|
|
28
29
|
* Append-only deduplicated string table.
|
|
29
30
|
*
|
|
30
|
-
* Limits:
|
|
31
|
+
* Limits: a string holds at most `MAX_STRING_BYTES` (65,535) UTF-8 bytes, and the table
|
|
32
|
+
* holds at most `MAX_UINT32_OFFSET` bytes in total. Larger input throws `OsmCapacityError`.
|
|
31
33
|
* Rebuilds reverse index lazily after transfer.
|
|
32
34
|
*/
|
|
33
35
|
export default class StringTable {
|
|
@@ -83,6 +85,18 @@ export default class StringTable {
|
|
|
83
85
|
if (existingIndex !== undefined) return existingIndex;
|
|
84
86
|
const startIndex = this.start.length;
|
|
85
87
|
const encoded = this.enc.encode(str);
|
|
88
|
+
assertCapacity(
|
|
89
|
+
"string-bytes",
|
|
90
|
+
encoded.length,
|
|
91
|
+
MAX_STRING_BYTES,
|
|
92
|
+
() => `String "${str.slice(0, 40)}…"`,
|
|
93
|
+
);
|
|
94
|
+
assertCapacity(
|
|
95
|
+
"total-string-bytes",
|
|
96
|
+
this.bytes.length + encoded.length,
|
|
97
|
+
MAX_UINT32_OFFSET,
|
|
98
|
+
() => "The string table",
|
|
99
|
+
);
|
|
86
100
|
this.start.push(this.bytes.length);
|
|
87
101
|
this.bytes.pushMany(encoded);
|
|
88
102
|
this.count.push(encoded.length);
|
package/src/tags.ts
CHANGED
|
@@ -12,16 +12,23 @@ import type { ContentHasher } from "@osmix/shared/content-hasher";
|
|
|
12
12
|
import type { OsmTags } from "@osmix/types";
|
|
13
13
|
|
|
14
14
|
import StringTable from "./stringtable.ts";
|
|
15
|
-
import { type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays.ts";
|
|
15
|
+
import { BufferConstructor, type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays.ts";
|
|
16
|
+
|
|
17
|
+
const TAG_RANK_BLOCK_SIZE = 256;
|
|
18
|
+
const TAG_BITS_PER_WORD = 32;
|
|
16
19
|
|
|
17
20
|
/**
|
|
18
21
|
* Serializable state for worker transfer.
|
|
19
22
|
*/
|
|
20
23
|
export interface TagsTransferables<T extends BufferType = BufferType> {
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
|
|
24
|
+
/** Exact number of entities represented by this tag index. */
|
|
25
|
+
tagEntityCount: number;
|
|
26
|
+
/** One presence bit per entity. */
|
|
27
|
+
taggedEntityBits: T;
|
|
28
|
+
/** Tagged-entity prefix count at each 256-entity boundary. */
|
|
29
|
+
tagRankCheckpoints: T;
|
|
30
|
+
/** Tag offsets for tagged entities, including a final sentinel. */
|
|
31
|
+
tagOffsets: T;
|
|
25
32
|
/** Flattened tag key indices. */
|
|
26
33
|
tagKeys: T;
|
|
27
34
|
/** Flattened tag value indices. */
|
|
@@ -38,7 +45,6 @@ export interface TagsTransferables<T extends BufferType = BufferType> {
|
|
|
38
45
|
/**
|
|
39
46
|
* Bidirectional tag storage.
|
|
40
47
|
*
|
|
41
|
-
* Limits: Max 255 tags per entity.
|
|
42
48
|
* Note: String indices reference a shared `StringTable`.
|
|
43
49
|
*/
|
|
44
50
|
export class Tags {
|
|
@@ -46,10 +52,14 @@ export class Tags {
|
|
|
46
52
|
private stringTable: StringTable = new StringTable();
|
|
47
53
|
|
|
48
54
|
// ─── Entity → Tag Lookup ───────────────────────────────────────────────────
|
|
49
|
-
/**
|
|
50
|
-
private
|
|
51
|
-
/**
|
|
52
|
-
private
|
|
55
|
+
/** Exact number of entities represented by this tag index. */
|
|
56
|
+
private entityCount = 0;
|
|
57
|
+
/** One presence bit per entity. */
|
|
58
|
+
private taggedEntityBits: Uint32Array;
|
|
59
|
+
/** Tagged-entity prefix count at each 256-entity boundary. */
|
|
60
|
+
private tagRankCheckpoints: Uint32Array;
|
|
61
|
+
/** Tag offsets for tagged entities, including a final sentinel. */
|
|
62
|
+
private tagOffsets: RTA<Uint32Array>;
|
|
53
63
|
/** All tag key string indices, concatenated */
|
|
54
64
|
private tagKeys: RTA<Uint32Array>;
|
|
55
65
|
/** All tag value string indices, concatenated (parallel to tagKeys) */
|
|
@@ -73,6 +83,12 @@ export class Tags {
|
|
|
73
83
|
*/
|
|
74
84
|
private keyEntityIndexBuilder = new Map<number, number[]>();
|
|
75
85
|
|
|
86
|
+
/** Tagged entity indexes collected during ingestion and released on finalization. */
|
|
87
|
+
private taggedEntityIndexBuilder: RTA<Uint32Array> | null;
|
|
88
|
+
|
|
89
|
+
/** Largest tagged entity index seen so far; enforces ascending ingestion order. */
|
|
90
|
+
private lastTaggedEntityIndex = -1;
|
|
91
|
+
|
|
76
92
|
/** Whether buildIndex() has been called */
|
|
77
93
|
private indexBuilt = false;
|
|
78
94
|
|
|
@@ -82,22 +98,27 @@ export class Tags {
|
|
|
82
98
|
constructor(stringTable: StringTable, transferables?: TagsTransferables) {
|
|
83
99
|
this.stringTable = stringTable;
|
|
84
100
|
if (transferables) {
|
|
85
|
-
this.
|
|
86
|
-
this.
|
|
101
|
+
this.entityCount = transferables.tagEntityCount;
|
|
102
|
+
this.taggedEntityBits = new Uint32Array(transferables.taggedEntityBits);
|
|
103
|
+
this.tagRankCheckpoints = new Uint32Array(transferables.tagRankCheckpoints);
|
|
104
|
+
this.tagOffsets = RTA.from(Uint32Array, transferables.tagOffsets);
|
|
87
105
|
this.tagKeys = RTA.from(Uint32Array, transferables.tagKeys);
|
|
88
106
|
this.tagVals = RTA.from(Uint32Array, transferables.tagVals);
|
|
89
107
|
this.keyEntities = RTA.from(Uint32Array, transferables.keyEntities);
|
|
90
108
|
this.keyIndexStart = RTA.from(Uint32Array, transferables.keyIndexStart);
|
|
91
109
|
this.keyIndexCount = RTA.from(Uint32Array, transferables.keyIndexCount);
|
|
110
|
+
this.taggedEntityIndexBuilder = null;
|
|
92
111
|
this.indexBuilt = true;
|
|
93
112
|
} else {
|
|
94
|
-
this.
|
|
95
|
-
this.
|
|
113
|
+
this.taggedEntityBits = new Uint32Array(new BufferConstructor(0));
|
|
114
|
+
this.tagRankCheckpoints = new Uint32Array(new BufferConstructor(0));
|
|
115
|
+
this.tagOffsets = new RTA(Uint32Array);
|
|
96
116
|
this.tagKeys = new RTA(Uint32Array);
|
|
97
117
|
this.tagVals = new RTA(Uint32Array);
|
|
98
118
|
this.keyEntities = new RTA(Uint32Array);
|
|
99
119
|
this.keyIndexStart = new RTA(Uint32Array);
|
|
100
120
|
this.keyIndexCount = new RTA(Uint32Array);
|
|
121
|
+
this.taggedEntityIndexBuilder = new RTA(Uint32Array);
|
|
101
122
|
}
|
|
102
123
|
}
|
|
103
124
|
|
|
@@ -122,10 +143,28 @@ export class Tags {
|
|
|
122
143
|
|
|
123
144
|
/**
|
|
124
145
|
* Add tags to an entity using key and value indexes.
|
|
146
|
+
*
|
|
147
|
+
* Tagged entities must be added in strictly ascending entity-index order and
|
|
148
|
+
* at most once per entity: the rank/popcount lookup orders tagged entities by
|
|
149
|
+
* index while `tagOffsets` records them in call order, so an out-of-order or
|
|
150
|
+
* repeated tagged index would silently misalign every later entity's tags.
|
|
151
|
+
* `Entities.addEntity` (append-only) satisfies this naturally.
|
|
125
152
|
*/
|
|
126
153
|
addTagKeysAndValues(index: number, keys: number[], values: number[]) {
|
|
127
|
-
this.
|
|
128
|
-
|
|
154
|
+
if (this.indexBuilt) throw Error("Tag index already built.");
|
|
155
|
+
if (keys.length !== values.length) throw Error("Tag keys and values must have equal length.");
|
|
156
|
+
if (index < 0) throw Error(`Invalid entity index: ${index}`);
|
|
157
|
+
this.entityCount = Math.max(this.entityCount, index + 1);
|
|
158
|
+
if (keys.length === 0) return;
|
|
159
|
+
|
|
160
|
+
if (index <= this.lastTaggedEntityIndex) {
|
|
161
|
+
throw Error(
|
|
162
|
+
`Tagged entity ${index} added out of order (last tagged entity: ${this.lastTaggedEntityIndex}). Tagged entities must be added once, in ascending index order.`,
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
this.lastTaggedEntityIndex = index;
|
|
166
|
+
this.taggedEntityIndexBuilder?.push(index);
|
|
167
|
+
this.tagOffsets.push(this.tagKeys.length);
|
|
129
168
|
this.tagKeys.pushMany(keys);
|
|
130
169
|
this.tagVals.pushMany(values);
|
|
131
170
|
|
|
@@ -146,8 +185,39 @@ export class Tags {
|
|
|
146
185
|
* Must be called before `hasKey()`.
|
|
147
186
|
*/
|
|
148
187
|
buildIndex() {
|
|
149
|
-
this.
|
|
150
|
-
|
|
188
|
+
if (this.indexBuilt) return;
|
|
189
|
+
|
|
190
|
+
const taggedEntityIndexes = this.taggedEntityIndexBuilder;
|
|
191
|
+
if (!taggedEntityIndexes) throw Error("Tagged entity builder is unavailable.");
|
|
192
|
+
const bitWordCount = Math.ceil(this.entityCount / TAG_BITS_PER_WORD);
|
|
193
|
+
this.taggedEntityBits = new Uint32Array(
|
|
194
|
+
new BufferConstructor(bitWordCount * Uint32Array.BYTES_PER_ELEMENT),
|
|
195
|
+
);
|
|
196
|
+
for (let i = 0; i < taggedEntityIndexes.length; i++) {
|
|
197
|
+
const entityIndex = taggedEntityIndexes.at(i);
|
|
198
|
+
const wordIndex = entityIndex >>> 5;
|
|
199
|
+
this.taggedEntityBits[wordIndex] =
|
|
200
|
+
(this.taggedEntityBits[wordIndex] ?? 0) | (1 << (entityIndex & 31));
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const checkpointCount = Math.ceil(this.entityCount / TAG_RANK_BLOCK_SIZE) + 1;
|
|
204
|
+
this.tagRankCheckpoints = new Uint32Array(
|
|
205
|
+
new BufferConstructor(checkpointCount * Uint32Array.BYTES_PER_ELEMENT),
|
|
206
|
+
);
|
|
207
|
+
let taggedIndex = 0;
|
|
208
|
+
for (let checkpoint = 0; checkpoint < checkpointCount; checkpoint++) {
|
|
209
|
+
const entityBoundary = checkpoint * TAG_RANK_BLOCK_SIZE;
|
|
210
|
+
while (
|
|
211
|
+
taggedIndex < taggedEntityIndexes.length &&
|
|
212
|
+
taggedEntityIndexes.at(taggedIndex) < entityBoundary
|
|
213
|
+
) {
|
|
214
|
+
taggedIndex++;
|
|
215
|
+
}
|
|
216
|
+
this.tagRankCheckpoints[checkpoint] = taggedIndex;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
this.tagOffsets.push(this.tagKeys.length);
|
|
220
|
+
this.tagOffsets.compact();
|
|
151
221
|
this.tagKeys.compact();
|
|
152
222
|
this.tagVals.compact();
|
|
153
223
|
|
|
@@ -158,9 +228,11 @@ export class Tags {
|
|
|
158
228
|
this.keyEntities.pushMany(entityIndexes);
|
|
159
229
|
}
|
|
160
230
|
|
|
231
|
+
this.keyEntities.compact();
|
|
161
232
|
this.keyIndexStart.compact();
|
|
162
233
|
this.keyIndexCount.compact();
|
|
163
234
|
this.keyEntityIndexBuilder.clear();
|
|
235
|
+
this.taggedEntityIndexBuilder = null;
|
|
164
236
|
|
|
165
237
|
this.indexBuilt = true;
|
|
166
238
|
}
|
|
@@ -176,16 +248,41 @@ export class Tags {
|
|
|
176
248
|
* Get the number of tags for an entity.
|
|
177
249
|
*/
|
|
178
250
|
cardinality(index: number): number {
|
|
179
|
-
|
|
251
|
+
const taggedIndex = this.getTaggedEntityIndex(index);
|
|
252
|
+
if (taggedIndex === -1) return 0;
|
|
253
|
+
return this.tagOffsets.at(taggedIndex + 1) - this.tagOffsets.at(taggedIndex);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Number of entities that have at least one tag. */
|
|
257
|
+
get taggedEntityCount(): number {
|
|
258
|
+
return this.indexBuilt
|
|
259
|
+
? Math.max(0, this.tagOffsets.length - 1)
|
|
260
|
+
: (this.taggedEntityIndexBuilder?.length ?? 0);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Iterate tagged entity indexes without allocating an intermediate array. */
|
|
264
|
+
*taggedEntityIndexes(): Generator<number> {
|
|
265
|
+
if (!this.indexBuilt) throw Error("Tag index not built.");
|
|
266
|
+
for (let wordIndex = 0; wordIndex < this.taggedEntityBits.length; wordIndex++) {
|
|
267
|
+
let word = this.taggedEntityBits[wordIndex] ?? 0;
|
|
268
|
+
while (word !== 0) {
|
|
269
|
+
const lowestBit = word & -word;
|
|
270
|
+
const bit = 31 - Math.clz32(lowestBit);
|
|
271
|
+
const entityIndex = wordIndex * TAG_BITS_PER_WORD + bit;
|
|
272
|
+
if (entityIndex < this.entityCount) yield entityIndex;
|
|
273
|
+
word = (word & (word - 1)) >>> 0;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
180
276
|
}
|
|
181
277
|
|
|
182
278
|
/**
|
|
183
279
|
* Get the tags for an entity.
|
|
184
280
|
*/
|
|
185
281
|
getTags(index: number): OsmTags | undefined {
|
|
186
|
-
const
|
|
187
|
-
if (
|
|
188
|
-
const tagStart = this.
|
|
282
|
+
const taggedIndex = this.getTaggedEntityIndex(index);
|
|
283
|
+
if (taggedIndex === -1) return;
|
|
284
|
+
const tagStart = this.tagOffsets.at(taggedIndex);
|
|
285
|
+
const tagCount = this.tagOffsets.at(taggedIndex + 1) - tagStart;
|
|
189
286
|
const tagKeyIndexes = this.tagKeys.array.slice(tagStart, tagStart + tagCount);
|
|
190
287
|
const tagValIndexes = this.tagVals.array.slice(tagStart, tagStart + tagCount);
|
|
191
288
|
const tags: OsmTags = {};
|
|
@@ -225,7 +322,12 @@ export class Tags {
|
|
|
225
322
|
* Get all entity indexes that have a specific tag key.
|
|
226
323
|
*/
|
|
227
324
|
hasKey(keyIndex: number): number[] {
|
|
228
|
-
if (
|
|
325
|
+
if (
|
|
326
|
+
keyIndex < 0 ||
|
|
327
|
+
keyIndex >= this.keyIndexStart.length ||
|
|
328
|
+
keyIndex >= this.keyIndexCount.length
|
|
329
|
+
)
|
|
330
|
+
return [];
|
|
229
331
|
const start = this.keyIndexStart.at(keyIndex) ?? 0;
|
|
230
332
|
const count = this.keyIndexCount.at(keyIndex) ?? 0;
|
|
231
333
|
return Array.from(this.keyEntities.array.subarray(start, start + count));
|
|
@@ -245,8 +347,10 @@ export class Tags {
|
|
|
245
347
|
*/
|
|
246
348
|
transferables(): TagsTransferables {
|
|
247
349
|
return {
|
|
248
|
-
|
|
249
|
-
|
|
350
|
+
tagEntityCount: this.entityCount,
|
|
351
|
+
taggedEntityBits: this.taggedEntityBits.buffer,
|
|
352
|
+
tagRankCheckpoints: this.tagRankCheckpoints.buffer,
|
|
353
|
+
tagOffsets: this.tagOffsets.array.buffer,
|
|
250
354
|
tagKeys: this.tagKeys.array.buffer,
|
|
251
355
|
tagVals: this.tagVals.array.buffer,
|
|
252
356
|
keyEntities: this.keyEntities.array.buffer,
|
|
@@ -256,12 +360,17 @@ export class Tags {
|
|
|
256
360
|
}
|
|
257
361
|
|
|
258
362
|
/**
|
|
259
|
-
* Get the approximate memory requirements for a
|
|
363
|
+
* Get the approximate memory requirements for a tag index in bytes.
|
|
364
|
+
*
|
|
365
|
+
* @param tagCount - Total key/value pairs. Each costs 12 bytes: key, value and the
|
|
366
|
+
* reverse key → entity index. The small per-key index arrays are not counted.
|
|
260
367
|
*/
|
|
261
|
-
static getBytesRequired(
|
|
368
|
+
static getBytesRequired(entityCount: number, taggedEntityCount = entityCount, tagCount = 0) {
|
|
262
369
|
return (
|
|
263
|
-
|
|
264
|
-
|
|
370
|
+
Math.ceil(entityCount / TAG_BITS_PER_WORD) * Uint32Array.BYTES_PER_ELEMENT +
|
|
371
|
+
(Math.ceil(entityCount / TAG_RANK_BLOCK_SIZE) + 1) * Uint32Array.BYTES_PER_ELEMENT +
|
|
372
|
+
(taggedEntityCount + 1) * Uint32Array.BYTES_PER_ELEMENT +
|
|
373
|
+
tagCount * 3 * Uint32Array.BYTES_PER_ELEMENT // tagKeys + tagVals + keyEntities
|
|
265
374
|
);
|
|
266
375
|
}
|
|
267
376
|
|
|
@@ -280,9 +389,42 @@ export class Tags {
|
|
|
280
389
|
*/
|
|
281
390
|
updateHash(hasher: ContentHasher): ContentHasher {
|
|
282
391
|
return hasher
|
|
283
|
-
.
|
|
284
|
-
.update(this.
|
|
392
|
+
.updateNumber(this.entityCount)
|
|
393
|
+
.update(this.taggedEntityBits)
|
|
394
|
+
.update(this.tagOffsets.array)
|
|
285
395
|
.update(this.tagKeys.array)
|
|
286
396
|
.update(this.tagVals.array);
|
|
287
397
|
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Return the compact tagged-entity ordinal, or -1 when the entity has no tags.
|
|
401
|
+
*
|
|
402
|
+
* The ordinal is the entity's rank among tagged entities: read the cached
|
|
403
|
+
* prefix count at the nearest 256-entity checkpoint, popcount the presence
|
|
404
|
+
* words between the checkpoint and the entity's word, then popcount the bits
|
|
405
|
+
* below the entity within its own word. The result indexes `tagOffsets`.
|
|
406
|
+
*/
|
|
407
|
+
private getTaggedEntityIndex(index: number): number {
|
|
408
|
+
if (index < 0 || index >= this.entityCount) return -1;
|
|
409
|
+
const wordIndex = index >>> 5;
|
|
410
|
+
const word = this.taggedEntityBits[wordIndex] ?? 0;
|
|
411
|
+
const bit = index & 31;
|
|
412
|
+
if ((word & (1 << bit)) === 0) return -1;
|
|
413
|
+
|
|
414
|
+
const checkpoint = index >>> 8;
|
|
415
|
+
let rank = this.tagRankCheckpoints[checkpoint] ?? 0;
|
|
416
|
+
const checkpointWord = checkpoint * (TAG_RANK_BLOCK_SIZE / TAG_BITS_PER_WORD);
|
|
417
|
+
for (let i = checkpointWord; i < wordIndex; i++) {
|
|
418
|
+
rank += popcount(this.taggedEntityBits[i] ?? 0);
|
|
419
|
+
}
|
|
420
|
+
const lowerBitMask = bit === 0 ? 0 : 0xffffffff >>> (TAG_BITS_PER_WORD - bit);
|
|
421
|
+
return rank + popcount(word & lowerBitMask);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** Count set bits in an unsigned 32-bit integer. */
|
|
426
|
+
function popcount(value: number): number {
|
|
427
|
+
const pairs = value - ((value >>> 1) & 0x55555555);
|
|
428
|
+
const nibbles = (pairs & 0x33333333) + ((pairs >>> 2) & 0x33333333);
|
|
429
|
+
return (((nibbles + (nibbles >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24;
|
|
288
430
|
}
|