@osmix/core 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +54 -7
  3. package/dist/contracts.d.ts +2 -1
  4. package/dist/contracts.d.ts.map +1 -1
  5. package/dist/entities.d.ts +21 -1
  6. package/dist/entities.d.ts.map +1 -1
  7. package/dist/entities.js +52 -3
  8. package/dist/entities.js.map +1 -1
  9. package/dist/ids.d.ts +18 -5
  10. package/dist/ids.d.ts.map +1 -1
  11. package/dist/ids.js +68 -18
  12. package/dist/ids.js.map +1 -1
  13. package/dist/index.d.ts +6 -3
  14. package/dist/index.d.ts.map +1 -1
  15. package/dist/index.js +6 -3
  16. package/dist/index.js.map +1 -1
  17. package/dist/indirect-kd-index.d.ts +28 -0
  18. package/dist/indirect-kd-index.d.ts.map +1 -0
  19. package/dist/indirect-kd-index.js +163 -0
  20. package/dist/indirect-kd-index.js.map +1 -0
  21. package/dist/limits.d.ts +30 -0
  22. package/dist/limits.d.ts.map +1 -0
  23. package/dist/limits.js +51 -0
  24. package/dist/limits.js.map +1 -0
  25. package/dist/nodes.d.ts +28 -13
  26. package/dist/nodes.d.ts.map +1 -1
  27. package/dist/nodes.js +181 -51
  28. package/dist/nodes.js.map +1 -1
  29. package/dist/osm.d.ts +59 -3
  30. package/dist/osm.d.ts.map +1 -1
  31. package/dist/osm.js +31 -2
  32. package/dist/osm.js.map +1 -1
  33. package/dist/relations.d.ts +10 -4
  34. package/dist/relations.d.ts.map +1 -1
  35. package/dist/relations.js +66 -28
  36. package/dist/relations.js.map +1 -1
  37. package/dist/renumber.d.ts +23 -0
  38. package/dist/renumber.d.ts.map +1 -0
  39. package/dist/renumber.js +64 -0
  40. package/dist/renumber.js.map +1 -0
  41. package/dist/stringtable.d.ts +2 -1
  42. package/dist/stringtable.d.ts.map +1 -1
  43. package/dist/stringtable.js +5 -1
  44. package/dist/stringtable.js.map +1 -1
  45. package/dist/tags.d.ts +44 -11
  46. package/dist/tags.d.ts.map +1 -1
  47. package/dist/tags.js +153 -27
  48. package/dist/tags.js.map +1 -1
  49. package/dist/typed-arrays.d.ts +63 -48
  50. package/dist/typed-arrays.d.ts.map +1 -1
  51. package/dist/typed-arrays.js +184 -107
  52. package/dist/typed-arrays.js.map +1 -1
  53. package/dist/ways.d.ts +50 -2
  54. package/dist/ways.d.ts.map +1 -1
  55. package/dist/ways.js +212 -35
  56. package/dist/ways.js.map +1 -1
  57. package/package.json +11 -12
  58. package/src/contracts.ts +2 -1
  59. package/src/entities.ts +67 -4
  60. package/src/ids.ts +72 -22
  61. package/src/index.ts +13 -7
  62. package/src/indirect-kd-index.ts +212 -0
  63. package/src/limits.ts +73 -0
  64. package/src/nodes.ts +228 -54
  65. package/src/osm.ts +90 -4
  66. package/src/relations.ts +80 -30
  67. package/src/renumber.ts +80 -0
  68. package/src/stringtable.ts +15 -1
  69. package/src/tags.ts +173 -31
  70. package/src/typed-arrays.ts +225 -113
  71. package/src/ways.ts +233 -38
  72. package/test/entity-index-errors.test.ts +41 -0
  73. package/test/ids.bench.ts +11 -7
  74. package/test/ids.test.ts +38 -0
  75. package/test/limits.test.ts +83 -0
  76. package/test/no-shared-array-buffer.test.ts +41 -0
  77. package/test/nodes-spatial-index.test.ts +261 -0
  78. package/test/osm-bbox.test.ts +30 -0
  79. package/test/relations.bench.ts +9 -4
  80. package/test/relations.test.ts +52 -29
  81. package/test/renumber.test.ts +54 -0
  82. package/test/spatial-index-rebuild.test.ts +33 -0
  83. package/test/storage-compaction.test.ts +269 -0
  84. package/test/tags-search.test.ts +13 -0
  85. package/test/typed-arrays.test.ts +178 -59
  86. package/src/types/geokdbush.d.ts +0 -32
  87. package/src/types/kdbush.d.ts +0 -47
package/src/relations.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { buildRelationRings } from "@osmix/geo/relation-multipolygon";
2
2
  import type { OsmPbfRelation } from "@osmix/pbf";
3
3
  import { assertValue } from "@osmix/shared/assert";
4
+ import { BitSet } from "@osmix/shared/bit-set";
4
5
  import type { ContentHasher } from "@osmix/shared/content-hasher";
5
6
  import type {
6
7
  GeoBbox2D,
@@ -23,8 +24,9 @@ import { bboxFromLonLats } from "@osmix/types/utils";
23
24
  import Flatbush from "flatbush";
24
25
  import { around as geoAround } from "geoflatbush";
25
26
 
26
- import { Entities, type EntitiesTransferables } from "./entities.ts";
27
+ import { Entities, type EntitiesTransferables, isValidSpatialBbox } from "./entities.ts";
27
28
  import { type IdOrIndex, Ids } from "./ids.ts";
29
+ import { MAX_RELATION_MEMBERS, MAX_UINT32_OFFSET, assertCapacity } from "./limits.ts";
28
30
  import type { Nodes } from "./nodes.ts";
29
31
  import type StringTable from "./stringtable.ts";
30
32
  import { Tags } from "./tags.ts";
@@ -55,7 +57,7 @@ export class Relations extends Entities<OsmRelation> {
55
57
  private stringTable: StringTable;
56
58
 
57
59
  private memberStart: RTA<Uint32Array>;
58
- private memberCount: RTA<Uint16Array>; // Maximum 65,535 members per relation
60
+ private memberCount: RTA<Uint16Array>; // Maximum MAX_RELATION_MEMBERS (65,535) per relation
59
61
 
60
62
  // Store the ID of the member because relations have other relations as members.
61
63
  private memberRefs: RTA<Float64Array>;
@@ -74,8 +76,9 @@ export class Relations extends Entities<OsmRelation> {
74
76
  private nodes: Nodes;
75
77
  private ways: Ways;
76
78
 
77
- // Lazily computed relation way membership. Invalidated whenever relations change.
78
- private wayMemberIdsCache: ReadonlySet<number> | null = null;
79
+ // Lazily computed relation way membership keyed by way index. Invalidated whenever relations
80
+ // change, and rebuilt if the way count no longer matches.
81
+ private wayMemberCache: BitSet | null = null;
79
82
 
80
83
  /**
81
84
  * Create a new Relations index.
@@ -114,11 +117,28 @@ export class Relations extends Entities<OsmRelation> {
114
117
  this.stringTable = stringTable;
115
118
  }
116
119
 
120
+ /** Reject a relation whose member count would wrap `memberCount` or `memberStart`. */
121
+ private assertMemberCapacity(relationId: number, memberCount: number) {
122
+ assertCapacity(
123
+ "relation-members",
124
+ memberCount,
125
+ MAX_RELATION_MEMBERS,
126
+ () => `Relation ${relationId}`,
127
+ );
128
+ assertCapacity(
129
+ "total-relation-members",
130
+ this.memberRefs.length + memberCount,
131
+ MAX_UINT32_OFFSET,
132
+ () => `The dataset (at relation ${relationId})`,
133
+ );
134
+ }
135
+
117
136
  /**
118
137
  * Add a single relation to the index.
119
138
  */
120
139
  addRelation(relation: OsmRelation) {
121
- this.wayMemberIdsCache = null;
140
+ this.wayMemberCache = null;
141
+ this.assertMemberCapacity(relation.id, relation.members.length);
122
142
  const relationIndex = this.addEntity(relation.id, relation.tags ?? {});
123
143
  this.memberStart.push(this.memberRefs.length);
124
144
  this.memberCount.push(relation.members.length);
@@ -138,7 +158,7 @@ export class Relations extends Entities<OsmRelation> {
138
158
  blockStringIndexMap: Uint32Array,
139
159
  filter?: (relation: OsmRelation) => OsmRelation | null,
140
160
  ): number {
141
- this.wayMemberIdsCache = null;
161
+ this.wayMemberCache = null;
142
162
  const blockToStringTable = (k: number) => {
143
163
  const index = blockStringIndexMap[k];
144
164
  if (index === undefined) throw Error("Tag key not found");
@@ -190,6 +210,7 @@ export class Relations extends Entities<OsmRelation> {
190
210
  })
191
211
  : null;
192
212
  if (filter && filteredRelation === null) continue;
213
+ this.assertMemberCapacity(relation.id, filteredRelation?.members.length ?? memberRefs.length);
193
214
  added++;
194
215
 
195
216
  this.addEntity(relation.id, tagKeys, tagValues);
@@ -221,11 +242,17 @@ export class Relations extends Entities<OsmRelation> {
221
242
  * Build the spatial index for relations.
222
243
  * Handles nested relations by resolving all descendant nodes and ways.
223
244
  * If bbox data already exists (e.g., loaded from storage), reuses it.
245
+ * Returns the existing index when it is already built.
224
246
  */
225
247
  buildSpatialIndex() {
226
248
  if (!this.nodes.isReady()) throw Error("Node index is not ready.");
227
249
  if (!this.ways.isReady()) throw Error("Way index is not ready.");
228
- if (this.size === 0) return this.spatialIndex;
250
+ if (this.spatialIndexBuilt) return this.spatialIndex;
251
+ if (this.size === 0) {
252
+ this.spatialIndex = new Flatbush(1, 128, Float64Array, BufferConstructor);
253
+ this.spatialIndexBuilt = true;
254
+ return this.spatialIndex;
255
+ }
229
256
  console.time("RelationIndex.buildSpatialIndex");
230
257
 
231
258
  this.spatialIndex = new Flatbush(this.size, 128, Float64Array, BufferConstructor);
@@ -300,11 +327,12 @@ export class Relations extends Entities<OsmRelation> {
300
327
  if (ll) lls.push(ll);
301
328
  }
302
329
 
303
- // Collect coordinates from resolved ways
330
+ // Collect coordinates from resolved ways. Member ways may reference nodes
331
+ // outside a referentially incomplete extract, so skip unresolvable refs.
304
332
  for (const wayId of resolved.ways) {
305
333
  const wayIndex = this.ways.ids.getIndexFromId(wayId);
306
334
  if (wayIndex === -1) continue;
307
- const wayPositions = this.ways.getCoordinates(wayIndex);
335
+ const wayPositions = this.ways.getResolvedCoordinates(wayIndex);
308
336
  lls.push(...wayPositions);
309
337
  }
310
338
 
@@ -372,30 +400,35 @@ export class Relations extends Entities<OsmRelation> {
372
400
  }
373
401
 
374
402
  /**
375
- * Get all way IDs that are members of relations, including nested relations.
403
+ * Check whether a way is a member of any relation, including through nested relations.
376
404
  * Used to exclude these ways from individual rendering.
405
+ *
406
+ * Membership is computed once and cached as one bit per way index. Requires the way ID index.
377
407
  */
378
- getWayMemberIds(): ReadonlySet<number> {
379
- if (this.wayMemberIdsCache !== null) return this.wayMemberIdsCache;
408
+ isWayMember(wayIndex: number): boolean {
409
+ return this.getWayMembers().has(wayIndex);
410
+ }
380
411
 
381
- const wayIds = new Set<number>();
412
+ private getWayMembers(): BitSet {
413
+ if (this.wayMemberCache !== null && this.wayMemberCache.size === this.ways.size) {
414
+ return this.wayMemberCache;
415
+ }
416
+
417
+ const members = new BitSet(this.ways.size);
418
+ const getRelation = (relId: number) => {
419
+ const relIndex = this.ids.getIndexFromId(relId);
420
+ if (relIndex === -1) return null;
421
+ return this.getByIndex(relIndex);
422
+ };
382
423
  for (let i = 0; i < this.size; i++) {
383
- const relation = this.getByIndex(i);
384
- const resolved = resolveRelationMembers(
385
- relation,
386
- (relId) => {
387
- const relIndex = this.ids.getIndexFromId(relId);
388
- if (relIndex === -1) return null;
389
- return this.getByIndex(relIndex);
390
- },
391
- 10, // max depth
392
- );
424
+ const resolved = resolveRelationMembers(this.getByIndex(i), getRelation, 10);
393
425
  for (const wayId of resolved.ways) {
394
- wayIds.add(wayId);
426
+ const wayIndex = this.ways.ids.getIndexFromId(wayId);
427
+ if (wayIndex !== -1) members.add(wayIndex);
395
428
  }
396
429
  }
397
- this.wayMemberIdsCache = wayIds;
398
- return this.wayMemberIdsCache;
430
+ this.wayMemberCache = members;
431
+ return members;
399
432
  }
400
433
 
401
434
  /**
@@ -457,7 +490,13 @@ export class Relations extends Entities<OsmRelation> {
457
490
  */
458
491
  intersects(bbox: GeoBbox2D, filterFn?: (index: number) => boolean): number[] {
459
492
  if (this.size === 0) return [];
460
- return this.spatialIndex.search(bbox[0], bbox[1], bbox[2], bbox[3], filterFn);
493
+ // A relation with no resolvable member geometry stores an inverted bbox.
494
+ // Flatbush's contained-node fast path skips per-leaf intersection tests,
495
+ // so filter inverted boxes out explicitly.
496
+ return this.spatialIndex.search(bbox[0], bbox[1], bbox[2], bbox[3], (index, x0, y0, x1, y1) => {
497
+ if (!isValidSpatialBbox(x0, y0, x1, y1)) return false;
498
+ return filterFn ? filterFn(index) : true;
499
+ });
461
500
  }
462
501
 
463
502
  /**
@@ -471,7 +510,15 @@ export class Relations extends Entities<OsmRelation> {
471
510
  neighbors(lon: number, lat: number, maxResults?: number, maxDistanceKm?: number): number[] {
472
511
  if (this.size === 0) return [];
473
512
  // Use geoflatbush for proper geographic distance calculations
474
- return geoAround(this.spatialIndex, lon, lat, maxResults, maxDistanceKm);
513
+ return geoAround(this.spatialIndex, lon, lat, maxResults, maxDistanceKm, (index) => {
514
+ const offset = index * 4;
515
+ return isValidSpatialBbox(
516
+ this.bbox.at(offset),
517
+ this.bbox.at(offset + 1),
518
+ this.bbox.at(offset + 2),
519
+ this.bbox.at(offset + 3),
520
+ );
521
+ });
475
522
  }
476
523
 
477
524
  /**
@@ -498,7 +545,7 @@ export class Relations extends Entities<OsmRelation> {
498
545
  /**
499
546
  * Get the approximate memory requirements for a given number of relations in bytes.
500
547
  */
501
- static getBytesRequired(count: number) {
548
+ static getBytesRequired(count: number, memberCount = 0, tagCount = 0) {
502
549
  if (count === 0) return 0;
503
550
  // Approximate members per relation
504
551
  let numNodes = count;
@@ -513,9 +560,12 @@ export class Relations extends Entities<OsmRelation> {
513
560
 
514
561
  return (
515
562
  Ids.getBytesRequired(count) +
516
- Tags.getBytesRequired(count) +
563
+ Tags.getBytesRequired(count, count, tagCount) +
517
564
  count * Uint32Array.BYTES_PER_ELEMENT + // memberStart
518
565
  count * Uint16Array.BYTES_PER_ELEMENT + // memberCount
566
+ memberCount * Float64Array.BYTES_PER_ELEMENT + // memberRefs
567
+ memberCount * Uint8Array.BYTES_PER_ELEMENT + // memberTypes
568
+ memberCount * Uint32Array.BYTES_PER_ELEMENT + // memberRoles
519
569
  count * 4 * Float64Array.BYTES_PER_ELEMENT + // bbox
520
570
  spatialIndexBytes
521
571
  );
@@ -0,0 +1,80 @@
1
+ import { Osm } from "./osm.ts";
2
+
3
+ /** Old ID → new ID for each renumbered entity, by type. JSON- and structured-clone-safe. */
4
+ export interface OsmIdMap {
5
+ nodes: Record<number, number>;
6
+ ways: Record<number, number>;
7
+ relations: Record<number, number>;
8
+ }
9
+
10
+ /** New positive IDs for one type's negative IDs, after its highest positive ID, in −1, −2, … order. */
11
+ function positiveIds(ids: { size: number; at(index: number): number }): Map<number, number> {
12
+ let max = 0;
13
+ const negative: number[] = [];
14
+ for (let index = 0; index < ids.size; index++) {
15
+ const id = ids.at(index);
16
+ if (id < 0) negative.push(id);
17
+ else max = Math.max(max, id);
18
+ }
19
+ negative.sort((a, b) => b - a);
20
+ return new Map(negative.map((id, index) => [id, max + index + 1]));
21
+ }
22
+
23
+ /**
24
+ * A copy of `osm` whose negative node, way and relation IDs are replaced by positive ones, for
25
+ * tools that reject negative IDs. In OSM files a negative ID conventionally marks a new, not yet
26
+ * uploaded entity, so only do this when the consumer needs it. Each type gets IDs after its own
27
+ * highest positive ID, in −1, −2, … order, so the result is deterministic. Way refs and relation
28
+ * members follow. `osm` is unchanged; the copy has its indexes built.
29
+ */
30
+ export function renumberNegativeIds(osm: Osm, options: { id?: string } = {}) {
31
+ const nodeIds = positiveIds(osm.nodes.ids);
32
+ const wayIds = positiveIds(osm.ways.ids);
33
+ const relationIds = positiveIds(osm.relations.ids);
34
+ const mapped = { node: nodeIds, way: wayIds, relation: relationIds } as const;
35
+ const renumbered = new Osm({ id: options.id ?? osm.id, header: osm.header });
36
+ for (const node of osm.nodes.sorted()) {
37
+ renumbered.nodes.addNode({ ...node, id: nodeIds.get(node.id) ?? node.id });
38
+ }
39
+ for (const way of osm.ways.sorted()) {
40
+ renumbered.ways.addWay({
41
+ ...way,
42
+ id: wayIds.get(way.id) ?? way.id,
43
+ refs: way.refs.map((ref) => nodeIds.get(ref) ?? ref),
44
+ });
45
+ }
46
+ for (const relation of osm.relations.sorted()) {
47
+ renumbered.relations.addRelation({
48
+ ...relation,
49
+ id: relationIds.get(relation.id) ?? relation.id,
50
+ members: relation.members.map((member) => ({
51
+ ...member,
52
+ ref: mapped[member.type].get(member.ref) ?? member.ref,
53
+ })),
54
+ });
55
+ }
56
+ renumbered.buildIndexes();
57
+ renumbered.buildSpatialIndexes();
58
+ return { osm: renumbered, idMap: toIdMap(nodeIds, wayIds, relationIds) };
59
+ }
60
+
61
+ function toIdMap(
62
+ nodes: Map<number, number>,
63
+ ways: Map<number, number>,
64
+ relations: Map<number, number>,
65
+ ): OsmIdMap {
66
+ return {
67
+ nodes: Object.fromEntries(nodes),
68
+ ways: Object.fromEntries(ways),
69
+ relations: Object.fromEntries(relations),
70
+ };
71
+ }
72
+
73
+ /** The ID map `renumberNegativeIds(osm)` would apply, without copying the data. */
74
+ export function negativeIdMap(osm: Osm): OsmIdMap {
75
+ return toIdMap(
76
+ positiveIds(osm.nodes.ids),
77
+ positiveIds(osm.ways.ids),
78
+ positiveIds(osm.relations.ids),
79
+ );
80
+ }
@@ -10,6 +10,7 @@
10
10
  import type { OsmPbfStringTable } from "@osmix/pbf";
11
11
  import type { ContentHasher } from "@osmix/shared/content-hasher";
12
12
 
13
+ import { MAX_STRING_BYTES, MAX_UINT32_OFFSET, assertCapacity } from "./limits.ts";
13
14
  import { type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays.ts";
14
15
 
15
16
  /**
@@ -27,7 +28,8 @@ export interface StringTableTransferables<T extends BufferType = BufferType> {
27
28
  /**
28
29
  * Append-only deduplicated string table.
29
30
  *
30
- * Limits: Max string length 65,535 bytes.
31
+ * Limits: a string holds at most `MAX_STRING_BYTES` (65,535) UTF-8 bytes, and the table
32
+ * holds at most `MAX_UINT32_OFFSET` bytes in total. Larger input throws `OsmCapacityError`.
31
33
  * Rebuilds reverse index lazily after transfer.
32
34
  */
33
35
  export default class StringTable {
@@ -83,6 +85,18 @@ export default class StringTable {
83
85
  if (existingIndex !== undefined) return existingIndex;
84
86
  const startIndex = this.start.length;
85
87
  const encoded = this.enc.encode(str);
88
+ assertCapacity(
89
+ "string-bytes",
90
+ encoded.length,
91
+ MAX_STRING_BYTES,
92
+ () => `String "${str.slice(0, 40)}…"`,
93
+ );
94
+ assertCapacity(
95
+ "total-string-bytes",
96
+ this.bytes.length + encoded.length,
97
+ MAX_UINT32_OFFSET,
98
+ () => "The string table",
99
+ );
86
100
  this.start.push(this.bytes.length);
87
101
  this.bytes.pushMany(encoded);
88
102
  this.count.push(encoded.length);
package/src/tags.ts CHANGED
@@ -12,16 +12,23 @@ import type { ContentHasher } from "@osmix/shared/content-hasher";
12
12
  import type { OsmTags } from "@osmix/types";
13
13
 
14
14
  import StringTable from "./stringtable.ts";
15
- import { type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays.ts";
15
+ import { BufferConstructor, type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays.ts";
16
+
17
+ const TAG_RANK_BLOCK_SIZE = 256;
18
+ const TAG_BITS_PER_WORD = 32;
16
19
 
17
20
  /**
18
21
  * Serializable state for worker transfer.
19
22
  */
20
23
  export interface TagsTransferables<T extends BufferType = BufferType> {
21
- /** Maps entity index → start position in tagKeys/tagVals. */
22
- tagStart: T;
23
- /** Maps entity index → number of tags. */
24
- tagCount: T;
24
+ /** Exact number of entities represented by this tag index. */
25
+ tagEntityCount: number;
26
+ /** One presence bit per entity. */
27
+ taggedEntityBits: T;
28
+ /** Tagged-entity prefix count at each 256-entity boundary. */
29
+ tagRankCheckpoints: T;
30
+ /** Tag offsets for tagged entities, including a final sentinel. */
31
+ tagOffsets: T;
25
32
  /** Flattened tag key indices. */
26
33
  tagKeys: T;
27
34
  /** Flattened tag value indices. */
@@ -38,7 +45,6 @@ export interface TagsTransferables<T extends BufferType = BufferType> {
38
45
  /**
39
46
  * Bidirectional tag storage.
40
47
  *
41
- * Limits: Max 255 tags per entity.
42
48
  * Note: String indices reference a shared `StringTable`.
43
49
  */
44
50
  export class Tags {
@@ -46,10 +52,14 @@ export class Tags {
46
52
  private stringTable: StringTable = new StringTable();
47
53
 
48
54
  // ─── Entity → Tag Lookup ───────────────────────────────────────────────────
49
- /** Maps entity index → start position in tagKeys/tagVals */
50
- private tagStart: RTA<Uint32Array>;
51
- /** Maps entity index → number of tags (max 255) */
52
- private tagCount: RTA<Uint8Array>;
55
+ /** Exact number of entities represented by this tag index. */
56
+ private entityCount = 0;
57
+ /** One presence bit per entity. */
58
+ private taggedEntityBits: Uint32Array;
59
+ /** Tagged-entity prefix count at each 256-entity boundary. */
60
+ private tagRankCheckpoints: Uint32Array;
61
+ /** Tag offsets for tagged entities, including a final sentinel. */
62
+ private tagOffsets: RTA<Uint32Array>;
53
63
  /** All tag key string indices, concatenated */
54
64
  private tagKeys: RTA<Uint32Array>;
55
65
  /** All tag value string indices, concatenated (parallel to tagKeys) */
@@ -73,6 +83,12 @@ export class Tags {
73
83
  */
74
84
  private keyEntityIndexBuilder = new Map<number, number[]>();
75
85
 
86
+ /** Tagged entity indexes collected during ingestion and released on finalization. */
87
+ private taggedEntityIndexBuilder: RTA<Uint32Array> | null;
88
+
89
+ /** Largest tagged entity index seen so far; enforces ascending ingestion order. */
90
+ private lastTaggedEntityIndex = -1;
91
+
76
92
  /** Whether buildIndex() has been called */
77
93
  private indexBuilt = false;
78
94
 
@@ -82,22 +98,27 @@ export class Tags {
82
98
  constructor(stringTable: StringTable, transferables?: TagsTransferables) {
83
99
  this.stringTable = stringTable;
84
100
  if (transferables) {
85
- this.tagStart = RTA.from(Uint32Array, transferables.tagStart);
86
- this.tagCount = RTA.from(Uint8Array, transferables.tagCount);
101
+ this.entityCount = transferables.tagEntityCount;
102
+ this.taggedEntityBits = new Uint32Array(transferables.taggedEntityBits);
103
+ this.tagRankCheckpoints = new Uint32Array(transferables.tagRankCheckpoints);
104
+ this.tagOffsets = RTA.from(Uint32Array, transferables.tagOffsets);
87
105
  this.tagKeys = RTA.from(Uint32Array, transferables.tagKeys);
88
106
  this.tagVals = RTA.from(Uint32Array, transferables.tagVals);
89
107
  this.keyEntities = RTA.from(Uint32Array, transferables.keyEntities);
90
108
  this.keyIndexStart = RTA.from(Uint32Array, transferables.keyIndexStart);
91
109
  this.keyIndexCount = RTA.from(Uint32Array, transferables.keyIndexCount);
110
+ this.taggedEntityIndexBuilder = null;
92
111
  this.indexBuilt = true;
93
112
  } else {
94
- this.tagStart = new RTA(Uint32Array);
95
- this.tagCount = new RTA(Uint8Array);
113
+ this.taggedEntityBits = new Uint32Array(new BufferConstructor(0));
114
+ this.tagRankCheckpoints = new Uint32Array(new BufferConstructor(0));
115
+ this.tagOffsets = new RTA(Uint32Array);
96
116
  this.tagKeys = new RTA(Uint32Array);
97
117
  this.tagVals = new RTA(Uint32Array);
98
118
  this.keyEntities = new RTA(Uint32Array);
99
119
  this.keyIndexStart = new RTA(Uint32Array);
100
120
  this.keyIndexCount = new RTA(Uint32Array);
121
+ this.taggedEntityIndexBuilder = new RTA(Uint32Array);
101
122
  }
102
123
  }
103
124
 
@@ -122,10 +143,28 @@ export class Tags {
122
143
 
123
144
  /**
124
145
  * Add tags to an entity using key and value indexes.
146
+ *
147
+ * Tagged entities must be added in strictly ascending entity-index order and
148
+ * at most once per entity: the rank/popcount lookup orders tagged entities by
149
+ * index while `tagOffsets` records them in call order, so an out-of-order or
150
+ * repeated tagged index would silently misalign every later entity's tags.
151
+ * `Entities.addEntity` (append-only) satisfies this naturally.
125
152
  */
126
153
  addTagKeysAndValues(index: number, keys: number[], values: number[]) {
127
- this.tagStart.set(index, this.tagKeys.length);
128
- this.tagCount.set(index, keys.length);
154
+ if (this.indexBuilt) throw Error("Tag index already built.");
155
+ if (keys.length !== values.length) throw Error("Tag keys and values must have equal length.");
156
+ if (index < 0) throw Error(`Invalid entity index: ${index}`);
157
+ this.entityCount = Math.max(this.entityCount, index + 1);
158
+ if (keys.length === 0) return;
159
+
160
+ if (index <= this.lastTaggedEntityIndex) {
161
+ throw Error(
162
+ `Tagged entity ${index} added out of order (last tagged entity: ${this.lastTaggedEntityIndex}). Tagged entities must be added once, in ascending index order.`,
163
+ );
164
+ }
165
+ this.lastTaggedEntityIndex = index;
166
+ this.taggedEntityIndexBuilder?.push(index);
167
+ this.tagOffsets.push(this.tagKeys.length);
129
168
  this.tagKeys.pushMany(keys);
130
169
  this.tagVals.pushMany(values);
131
170
 
@@ -146,8 +185,39 @@ export class Tags {
146
185
  * Must be called before `hasKey()`.
147
186
  */
148
187
  buildIndex() {
149
- this.tagStart.compact();
150
- this.tagCount.compact();
188
+ if (this.indexBuilt) return;
189
+
190
+ const taggedEntityIndexes = this.taggedEntityIndexBuilder;
191
+ if (!taggedEntityIndexes) throw Error("Tagged entity builder is unavailable.");
192
+ const bitWordCount = Math.ceil(this.entityCount / TAG_BITS_PER_WORD);
193
+ this.taggedEntityBits = new Uint32Array(
194
+ new BufferConstructor(bitWordCount * Uint32Array.BYTES_PER_ELEMENT),
195
+ );
196
+ for (let i = 0; i < taggedEntityIndexes.length; i++) {
197
+ const entityIndex = taggedEntityIndexes.at(i);
198
+ const wordIndex = entityIndex >>> 5;
199
+ this.taggedEntityBits[wordIndex] =
200
+ (this.taggedEntityBits[wordIndex] ?? 0) | (1 << (entityIndex & 31));
201
+ }
202
+
203
+ const checkpointCount = Math.ceil(this.entityCount / TAG_RANK_BLOCK_SIZE) + 1;
204
+ this.tagRankCheckpoints = new Uint32Array(
205
+ new BufferConstructor(checkpointCount * Uint32Array.BYTES_PER_ELEMENT),
206
+ );
207
+ let taggedIndex = 0;
208
+ for (let checkpoint = 0; checkpoint < checkpointCount; checkpoint++) {
209
+ const entityBoundary = checkpoint * TAG_RANK_BLOCK_SIZE;
210
+ while (
211
+ taggedIndex < taggedEntityIndexes.length &&
212
+ taggedEntityIndexes.at(taggedIndex) < entityBoundary
213
+ ) {
214
+ taggedIndex++;
215
+ }
216
+ this.tagRankCheckpoints[checkpoint] = taggedIndex;
217
+ }
218
+
219
+ this.tagOffsets.push(this.tagKeys.length);
220
+ this.tagOffsets.compact();
151
221
  this.tagKeys.compact();
152
222
  this.tagVals.compact();
153
223
 
@@ -158,9 +228,11 @@ export class Tags {
158
228
  this.keyEntities.pushMany(entityIndexes);
159
229
  }
160
230
 
231
+ this.keyEntities.compact();
161
232
  this.keyIndexStart.compact();
162
233
  this.keyIndexCount.compact();
163
234
  this.keyEntityIndexBuilder.clear();
235
+ this.taggedEntityIndexBuilder = null;
164
236
 
165
237
  this.indexBuilt = true;
166
238
  }
@@ -176,16 +248,41 @@ export class Tags {
176
248
  * Get the number of tags for an entity.
177
249
  */
178
250
  cardinality(index: number): number {
179
- return this.tagCount.at(index) ?? 0;
251
+ const taggedIndex = this.getTaggedEntityIndex(index);
252
+ if (taggedIndex === -1) return 0;
253
+ return this.tagOffsets.at(taggedIndex + 1) - this.tagOffsets.at(taggedIndex);
254
+ }
255
+
256
+ /** Number of entities that have at least one tag. */
257
+ get taggedEntityCount(): number {
258
+ return this.indexBuilt
259
+ ? Math.max(0, this.tagOffsets.length - 1)
260
+ : (this.taggedEntityIndexBuilder?.length ?? 0);
261
+ }
262
+
263
+ /** Iterate tagged entity indexes without allocating an intermediate array. */
264
+ *taggedEntityIndexes(): Generator<number> {
265
+ if (!this.indexBuilt) throw Error("Tag index not built.");
266
+ for (let wordIndex = 0; wordIndex < this.taggedEntityBits.length; wordIndex++) {
267
+ let word = this.taggedEntityBits[wordIndex] ?? 0;
268
+ while (word !== 0) {
269
+ const lowestBit = word & -word;
270
+ const bit = 31 - Math.clz32(lowestBit);
271
+ const entityIndex = wordIndex * TAG_BITS_PER_WORD + bit;
272
+ if (entityIndex < this.entityCount) yield entityIndex;
273
+ word = (word & (word - 1)) >>> 0;
274
+ }
275
+ }
180
276
  }
181
277
 
182
278
  /**
183
279
  * Get the tags for an entity.
184
280
  */
185
281
  getTags(index: number): OsmTags | undefined {
186
- const tagCount = this.tagCount.at(index) ?? 0;
187
- if (tagCount === 0) return;
188
- const tagStart = this.tagStart.at(index) ?? 0;
282
+ const taggedIndex = this.getTaggedEntityIndex(index);
283
+ if (taggedIndex === -1) return;
284
+ const tagStart = this.tagOffsets.at(taggedIndex);
285
+ const tagCount = this.tagOffsets.at(taggedIndex + 1) - tagStart;
189
286
  const tagKeyIndexes = this.tagKeys.array.slice(tagStart, tagStart + tagCount);
190
287
  const tagValIndexes = this.tagVals.array.slice(tagStart, tagStart + tagCount);
191
288
  const tags: OsmTags = {};
@@ -225,7 +322,12 @@ export class Tags {
225
322
  * Get all entity indexes that have a specific tag key.
226
323
  */
227
324
  hasKey(keyIndex: number): number[] {
228
- if (keyIndex < 0) return [];
325
+ if (
326
+ keyIndex < 0 ||
327
+ keyIndex >= this.keyIndexStart.length ||
328
+ keyIndex >= this.keyIndexCount.length
329
+ )
330
+ return [];
229
331
  const start = this.keyIndexStart.at(keyIndex) ?? 0;
230
332
  const count = this.keyIndexCount.at(keyIndex) ?? 0;
231
333
  return Array.from(this.keyEntities.array.subarray(start, start + count));
@@ -245,8 +347,10 @@ export class Tags {
245
347
  */
246
348
  transferables(): TagsTransferables {
247
349
  return {
248
- tagStart: this.tagStart.array.buffer,
249
- tagCount: this.tagCount.array.buffer,
350
+ tagEntityCount: this.entityCount,
351
+ taggedEntityBits: this.taggedEntityBits.buffer,
352
+ tagRankCheckpoints: this.tagRankCheckpoints.buffer,
353
+ tagOffsets: this.tagOffsets.array.buffer,
250
354
  tagKeys: this.tagKeys.array.buffer,
251
355
  tagVals: this.tagVals.array.buffer,
252
356
  keyEntities: this.keyEntities.array.buffer,
@@ -256,12 +360,17 @@ export class Tags {
256
360
  }
257
361
 
258
362
  /**
259
- * Get the approximate memory requirements for a given number of tags in bytes.
363
+ * Get the approximate memory requirements for a tag index in bytes.
364
+ *
365
+ * @param tagCount - Total key/value pairs. Each costs 12 bytes: key, value and the
366
+ * reverse key → entity index. The small per-key index arrays are not counted.
260
367
  */
261
- static getBytesRequired(count: number) {
368
+ static getBytesRequired(entityCount: number, taggedEntityCount = entityCount, tagCount = 0) {
262
369
  return (
263
- count * Uint32Array.BYTES_PER_ELEMENT + // tagStart
264
- count * Uint8Array.BYTES_PER_ELEMENT // tagCount
370
+ Math.ceil(entityCount / TAG_BITS_PER_WORD) * Uint32Array.BYTES_PER_ELEMENT +
371
+ (Math.ceil(entityCount / TAG_RANK_BLOCK_SIZE) + 1) * Uint32Array.BYTES_PER_ELEMENT +
372
+ (taggedEntityCount + 1) * Uint32Array.BYTES_PER_ELEMENT +
373
+ tagCount * 3 * Uint32Array.BYTES_PER_ELEMENT // tagKeys + tagVals + keyEntities
265
374
  );
266
375
  }
267
376
 
@@ -280,9 +389,42 @@ export class Tags {
280
389
  */
281
390
  updateHash(hasher: ContentHasher): ContentHasher {
282
391
  return hasher
283
- .update(this.tagStart.array)
284
- .update(this.tagCount.array)
392
+ .updateNumber(this.entityCount)
393
+ .update(this.taggedEntityBits)
394
+ .update(this.tagOffsets.array)
285
395
  .update(this.tagKeys.array)
286
396
  .update(this.tagVals.array);
287
397
  }
398
+
399
+ /**
400
+ * Return the compact tagged-entity ordinal, or -1 when the entity has no tags.
401
+ *
402
+ * The ordinal is the entity's rank among tagged entities: read the cached
403
+ * prefix count at the nearest 256-entity checkpoint, popcount the presence
404
+ * words between the checkpoint and the entity's word, then popcount the bits
405
+ * below the entity within its own word. The result indexes `tagOffsets`.
406
+ */
407
+ private getTaggedEntityIndex(index: number): number {
408
+ if (index < 0 || index >= this.entityCount) return -1;
409
+ const wordIndex = index >>> 5;
410
+ const word = this.taggedEntityBits[wordIndex] ?? 0;
411
+ const bit = index & 31;
412
+ if ((word & (1 << bit)) === 0) return -1;
413
+
414
+ const checkpoint = index >>> 8;
415
+ let rank = this.tagRankCheckpoints[checkpoint] ?? 0;
416
+ const checkpointWord = checkpoint * (TAG_RANK_BLOCK_SIZE / TAG_BITS_PER_WORD);
417
+ for (let i = checkpointWord; i < wordIndex; i++) {
418
+ rank += popcount(this.taggedEntityBits[i] ?? 0);
419
+ }
420
+ const lowerBitMask = bit === 0 ? 0 : 0xffffffff >>> (TAG_BITS_PER_WORD - bit);
421
+ return rank + popcount(word & lowerBitMask);
422
+ }
423
+ }
424
+
425
+ /** Count set bits in an unsigned 32-bit integer. */
426
+ function popcount(value: number): number {
427
+ const pairs = value - ((value >>> 1) & 0x55555555);
428
+ const nibbles = (pairs & 0x33333333) + ((pairs >>> 2) & 0x33333333);
429
+ return (((nibbles + (nibbles >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24;
288
430
  }