@osmix/core 0.0.2 → 0.1.1

package/src/stringtable.ts

@@ -1,44 +1,77 @@
-import type { OsmPbfBlock, OsmPbfStringTable } from "@osmix/pbf"
-import { type BufferType, ResizeableTypedArray } from "./typed-arrays"
+/**
+ * Deduplicated UTF-8 string storage.
+ *
+ * Strings are stored in a flat buffer and referenced by index.
+ * Supports fast deduplication (string→index) and lazy reconstruction (index→string).
+ *
+ * @module
+ */
+
+import type { OsmPbfStringTable } from "@osmix/pbf"
+import { type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays"
+import type { ContentHasher } from "./utils"
 
-export interface StringTableTransferables {
-	bytes: BufferType
-	start: BufferType
-	count: BufferType
+/**
+ * Serializable state for worker transfer.
+ */
+export interface StringTableTransferables<T extends BufferType = BufferType> {
+	/** Concatenated UTF-8 bytes. */
+	bytes: T
+	/** Maps string index → byte offset. */
+	start: T
+	/** Maps string index → byte length. */
+	count: T
 }
 
 /**
- * StringTable that can be used to store and retrieve strings as bytes.
+ * Append-only deduplicated string table.
+ *
+ * Limits: Max string length 65,535 bytes.
+ * Rebuilds reverse index lazily after transfer.
  */
 export default class StringTable {
+	/** UTF-8 encoder for string→bytes conversion */
 	private enc = new TextEncoder()
+	/** UTF-8 decoder for bytes→string conversion */
 	private dec = new TextDecoder() // UTF-8
 
-	// Serializable state
-	private bytes: ResizeableTypedArray<Uint8Array>
-	private start: ResizeableTypedArray<Uint32Array>
-	private count: ResizeableTypedArray<Uint16Array>
+	// ─── Serializable State ────────────────────────────────────────────────────
+	/** Concatenated UTF-8 bytes of all strings */
+	bytes: RTA<Uint8Array>
+	/** Maps string index → byte offset in bytes array */
+	start: RTA<Uint32Array>
+	/** Maps string index → byte length */
+	count: RTA<Uint16Array>
 
-	// Builder state
+	// ─── Builder State (string → index) ────────────────────────────────────────
+	/** Forward lookup: string → index (populated during add) */
 	private stringToIndex = new Map<string, number>()
 
-	// Retrieval state
-	private indexToString = new Map<number, string>()
+	/** Whether the reverse index has been built (lazy after transfer) */
+	private reverseIndexBuilt = false
 
-	static from({ bytes, start, count }: StringTableTransferables): StringTable {
-		const builder = new StringTable()
-		builder.bytes = ResizeableTypedArray.from(Uint8Array, bytes)
-		builder.start = ResizeableTypedArray.from(Uint32Array, start)
-		builder.count = ResizeableTypedArray.from(Uint16Array, count)
-		return builder
-	}
+	// ─── Retrieval Cache (index → string) ──────────────────────────────────────
+	/** Cache of decoded strings to avoid repeated UTF-8 decoding */
+	private indexToString = new Map<number, string>()
 
-	constructor() {
-		this.bytes = new ResizeableTypedArray(Uint8Array)
-		this.start = new ResizeableTypedArray(Uint32Array)
-		this.count = new ResizeableTypedArray(Uint16Array)
+	/**
+	 * Create a new StringTable.
+	 */
+	constructor(opts?: StringTableTransferables) {
+		this.bytes = opts?.bytes
+			? RTA.from(Uint8Array, opts.bytes)
+			: new RTA(Uint8Array)
+		this.start = opts?.start
+			? RTA.from(Uint32Array, opts.start)
+			: new RTA(Uint32Array)
+		this.count = opts?.count
+			? RTA.from(Uint16Array, opts.count)
+			: new RTA(Uint16Array)
 	}
 
+	/**
+	 * Get transferable objects for passing to another thread.
+	 */
 	transferables(): StringTableTransferables {
 		return {
 			bytes: this.bytes.array.buffer,
@@ -47,6 +80,9 @@ export default class StringTable {
 		}
 	}
 
+	/**
+	 * Add a string to the table and return its index.
+	 */
 	add(str: string): number {
 		const existingIndex = this.stringToIndex.get(str)
 		if (existingIndex !== undefined) return existingIndex
@@ -63,19 +99,24 @@ export default class StringTable {
 	 * Decode all the strings in a primitive block and add them to the string table.
 	 * Return a mapping of block index -> string table index
 	 */
-	createBlockIndexMap(block: OsmPbfBlock) {
-		const map = new Map<number, number>()
-		for (let i = 0; i < block.stringtable.length; i++) {
-			const bytesString = block.stringtable[i]
+	createBlockIndexMap(blockStringtable: OsmPbfStringTable) {
+		const index = new Uint32Array(blockStringtable.length)
+		for (let i = 0; i < blockStringtable.length; i++) {
+			const bytesString = blockStringtable[i]
 			const str = this.dec.decode(bytesString)
-			const index = this.add(str)
-			map.set(i, index)
+			const existingIndex = this.stringToIndex.get(str)
+			if (existingIndex !== undefined) {
+				index[i] = existingIndex
+				continue
+			}
+			index[i] = this.add(str)
 		}
-		return map
+		return index
 	}
 
 	/**
-	 * TextDecoder.decode() does not support SharedArrayBuffer, so we need to first copy to a normal ArrayBuffer before decoding.
+	 * Get a string by its index.
+	 * Caches results to avoid repeated UTF-8 decoding.
 	 */
 	get(index: number): string {
 		const string = this.indexToString.get(index)
@@ -89,27 +130,39 @@ export default class StringTable {
 		return decoded
 	}
 
+	/**
+	 * Get the raw UTF-8 bytes of a string.
+	 * Returns a subarray view (not a copy).
+	 */
 	getBytes(index: number): Uint8Array {
 		if (index < 0 || index >= this.length)
 			throw Error(`String index out of range: ${index}`)
 		const start = this.start.at(index)
 		const count = this.count.at(index)
-		// Important: subarray creates a *view*, not a copy; TextDecoder cannot read from it directly.
 		return this.bytes.array.subarray(start, start + count)
 	}
 
+	/** Number of strings in the table. */
 	get length() {
 		return this.start.length
 	}
 
+	/**
+	 * Finalize the string table by compacting internal arrays.
+	 *
+	 * This releases unused buffer capacity and marks the reverse index as built
+	 * (since all strings were added before this call).
+	 */
 	buildIndex() {
 		this.bytes.compact()
 		this.start.compact()
 		this.count.compact()
-
-		this.stringToIndex.clear()
+		this.reverseIndexBuilt = true
 	}
 
+	/**
+	 * Convert the string table to an OSM PBF string table.
+	 */
 	toOsmPbfStringTable(): OsmPbfStringTable {
 		const stringTable: OsmPbfStringTable = []
 		for (let i = 0; i < this.length; i++) {
@@ -117,4 +170,38 @@ export default class StringTable {
 		}
 		return stringTable
 	}
+
+	/**
+	 * Lazily build the reverse index (string → index).
+	 * Decodes all strings once to populate the lookup map.
+	 */
+	private ensureReverseIndex() {
+		if (this.reverseIndexBuilt) return
+		// Decode all strings and populate the forward lookup map
+		for (let i = 0; i < this.length; i++) {
+			this.stringToIndex.set(this.get(i), i)
+		}
+		this.reverseIndexBuilt = true
+	}
+
+	/**
+	 * Find the index of a string.
+	 */
+	find(str: string): number {
+		const existing = this.stringToIndex.get(str)
+		if (existing !== undefined) return existing
+		this.ensureReverseIndex()
+		return this.stringToIndex.get(str) ?? -1
+	}
+
+	/**
+	 * Update a ContentHasher with the string table's data.
+	 * Hashes the bytes, start offsets, and counts.
+	 */
+	updateHash(hasher: ContentHasher): ContentHasher {
+		return hasher
+			.update(this.bytes.array)
+			.update(this.start.array)
+			.update(this.count.array)
+	}
 }

package/src/tags.ts

@@ -1,54 +1,109 @@
-import type { OsmTags } from "@osmix/json"
-import type StringTable from "./stringtable"
-import { type BufferType, ResizeableTypedArray } from "./typed-arrays"
-
-export interface TagsTransferables {
-	tagStart: BufferType
-	tagCount: BufferType
-	tagKeys: BufferType
-	tagVals: BufferType
+/**
+ * Tag storage and lookup.
+ *
+ * Stores key=value pairs using string table indices. Supports:
+ * 1. **Entity → Tags**: Retrieve tags for a given entity.
+ * 2. **Key → Entities**: Find entities with a specific tag key.
+ *
+ * @module
+ */
+
+import type { OsmTags } from "@osmix/shared/types"
+import StringTable from "./stringtable"
+import { type BufferType, ResizeableTypedArray as RTA } from "./typed-arrays"
+import type { ContentHasher } from "./utils"
+
+/**
+ * Serializable state for worker transfer.
+ */
+export interface TagsTransferables<T extends BufferType = BufferType> {
+	/** Maps entity index → start position in tagKeys/tagVals. */
+	tagStart: T
+	/** Maps entity index → number of tags. */
+	tagCount: T
+	/** Flattened tag key indices. */
+	tagKeys: T
+	/** Flattened tag value indices. */
+	tagVals: T
+
+	/** Flattened entity indices for reverse key lookup. */
+	keyEntities: T
+	/** Maps key index → start position in keyEntities. */
+	keyIndexStart: T
+	/** Maps key index → count of entities with that key. */
+	keyIndexCount: T
 }
 
+/**
+ * Bidirectional tag storage.
+ *
+ * Limits: Max 255 tags per entity.
+ * Note: String indices reference a shared `StringTable`.
+ */
 export class Tags {
-	private stringTable: StringTable
-	private tagStart: ResizeableTypedArray<Uint32Array>
-	private tagCount: ResizeableTypedArray<Uint8Array>
-	private tagKeys: ResizeableTypedArray<Uint32Array>
-	private tagVals: ResizeableTypedArray<Uint32Array>
+	/** Reference to the shared string table for key/value storage */
+	private stringTable: StringTable = new StringTable()
 
-	private indexBuilt = false
+	// ─── Entity → Tag Lookup ───────────────────────────────────────────────────
+	/** Maps entity index → start position in tagKeys/tagVals */
+	private tagStart: RTA<Uint32Array>
+	/** Maps entity index → number of tags (max 255) */
+	private tagCount: RTA<Uint8Array>
+	/** All tag key string indices, concatenated */
+	private tagKeys: RTA<Uint32Array>
+	/** All tag value string indices, concatenated (parallel to tagKeys) */
+	private tagVals: RTA<Uint32Array>
 
-	static from(
-		stringTable: StringTable,
-		{ tagStart, tagCount, tagKeys, tagVals }: TagsTransferables,
-	) {
-		const tagIndex = new Tags(stringTable)
-		tagIndex.tagStart = ResizeableTypedArray.from(Uint32Array, tagStart)
-		tagIndex.tagCount = ResizeableTypedArray.from(Uint8Array, tagCount)
-		tagIndex.tagKeys = ResizeableTypedArray.from(Uint32Array, tagKeys)
-		tagIndex.tagVals = ResizeableTypedArray.from(Uint32Array, tagVals)
-		tagIndex.indexBuilt = true
-		return tagIndex
-	}
+	// ─── Key → Entity Reverse Lookup ───────────────────────────────────────────
+	/**
+	 * Flattened array of entity indices that have each tag key.
+	 * Indexed via keyIndexStart and keyIndexCount.
+	 */
+	private keyEntities: RTA<Uint32Array>
 
-	constructor(stringTable: StringTable) {
-		this.stringTable = stringTable
-		this.tagStart = new ResizeableTypedArray(Uint32Array)
-		this.tagCount = new ResizeableTypedArray(Uint8Array)
-		this.tagKeys = new ResizeableTypedArray(Uint32Array)
-		this.tagVals = new ResizeableTypedArray(Uint32Array)
-	}
+	/** Maps key string index → start position in keyEntities */
+	private keyIndexStart: RTA<Uint32Array>
+	/** Maps key string index → count of entities with that key */
+	private keyIndexCount: RTA<Uint32Array>
 
-	transferables() {
-		return {
-			tagStart: this.tagStart.array.buffer,
-			tagCount: this.tagCount.array.buffer,
-			tagKeys: this.tagKeys.array.buffer,
-			tagVals: this.tagVals.array.buffer,
+	/**
+	 * Temporary map used during ingestion to collect entity indices per key.
+	 * Converted to flat arrays during buildIndex() and then cleared.
+	 */
+	private keyEntityIndexBuilder = new Map<number, number[]>()
+
+	/** Whether buildIndex() has been called */
+	private indexBuilt = false
+
+	/**
+	 * Create a new Tags index.
+	 */
+	constructor(stringTable: StringTable, transferables?: TagsTransferables) {
+		this.stringTable = stringTable
+		if (transferables) {
+			this.tagStart = RTA.from(Uint32Array, transferables.tagStart)
+			this.tagCount = RTA.from(Uint8Array, transferables.tagCount)
+			this.tagKeys = RTA.from(Uint32Array, transferables.tagKeys)
+			this.tagVals = RTA.from(Uint32Array, transferables.tagVals)
+			this.keyEntities = RTA.from(Uint32Array, transferables.keyEntities)
+			this.keyIndexStart = RTA.from(Uint32Array, transferables.keyIndexStart)
+			this.keyIndexCount = RTA.from(Uint32Array, transferables.keyIndexCount)
+			this.indexBuilt = true
+		} else {
+			this.tagStart = new RTA(Uint32Array)
+			this.tagCount = new RTA(Uint8Array)
+			this.tagKeys = new RTA(Uint32Array)
+			this.tagVals = new RTA(Uint32Array)
+			this.keyEntities = new RTA(Uint32Array)
+			this.keyIndexStart = new RTA(Uint32Array)
+			this.keyIndexCount = new RTA(Uint32Array)
 		}
 	}
 
-	addTags(tags?: OsmTags) {
+	/**
+	 * Add tags to an entity.
+	 */
+	addTags(index: number, tags?: OsmTags): [number[], number[]] {
 		const tagKeys: number[] = []
 		const tagValues: number[] = []
 
@@ -59,32 +114,73 @@ export class Tags {
 			}
 		}
 
-		this.addTagKeysAndValues(tagKeys, tagValues)
+		this.addTagKeysAndValues(index, tagKeys, tagValues)
+
+		return [tagKeys, tagValues]
 	}
 
-	addTagKeysAndValues(keys: number[], values: number[]) {
-		this.tagStart.push(this.tagKeys.length)
-		this.tagCount.push(keys.length)
+	/**
+	 * Add tags to an entity using key and value indexes.
+	 */
+	addTagKeysAndValues(index: number, keys: number[], values: number[]) {
+		this.tagStart.set(index, this.tagKeys.length)
+		this.tagCount.set(index, keys.length)
 		this.tagKeys.pushMany(keys)
 		this.tagVals.pushMany(values)
+
+		keys.forEach((key) => {
+			const keyEntities = this.keyEntityIndexBuilder.get(key)
+			if (keyEntities) {
+				keyEntities.push(index)
+			} else {
+				this.keyEntityIndexBuilder.set(key, [index])
+			}
+		})
 	}
 
+	/**
+	 * Finalize the tag index.
+	 *
+	 * Compacts arrays and builds the reverse key→entity index.
+	 * Must be called before `hasKey()`.
+	 */
 	buildIndex() {
 		this.tagStart.compact()
 		this.tagCount.compact()
 		this.tagKeys.compact()
 		this.tagVals.compact()
+
+		// Convert the builder map to flat arrays for the reverse index
+		for (const [keyIndex, entityIndexes] of this.keyEntityIndexBuilder) {
+			this.keyIndexStart.set(keyIndex, this.keyEntities.length)
+			this.keyIndexCount.set(keyIndex, entityIndexes.length)
+			this.keyEntities.pushMany(entityIndexes)
+		}
+
+		this.keyIndexStart.compact()
+		this.keyIndexCount.compact()
+		this.keyEntityIndexBuilder.clear()
+
 		this.indexBuilt = true
 	}
 
-	get isReady() {
+	/**
+	 * Check if the index is built and ready for use.
+	 */
+	isReady() {
 		return this.indexBuilt
 	}
 
+	/**
+	 * Get the number of tags for an entity.
+	 */
 	cardinality(index: number): number {
 		return this.tagCount.at(index) ?? 0
 	}
 
+	/**
+	 * Get the tags for an entity.
+	 */
 	getTags(index: number): OsmTags | undefined {
 		const tagCount = this.tagCount.at(index) ?? 0
 		if (tagCount === 0) return
@@ -108,6 +204,9 @@ export class Tags {
 		return tags
 	}
 
+	/**
+	 * Get tags from key and value indexes.
+	 */
 	getTagsFromIndices(keys: number[], values: number[]): OsmTags {
 		const tags: OsmTags = {}
 		for (let i = 0; i < keys.length; i++) {
@@ -119,4 +218,79 @@ export class Tags {
 		}
 		return tags
 	}
+
+	/**
+	 * Find the index of a tag key.
+	 */
+	find(key: string): number {
+		return this.stringTable.find(key)
+	}
+
+	/**
+	 * Get all entity indexes that have a specific tag key.
+	 */
+	hasKey(keyIndex: number): number[] {
+		if (keyIndex < 0) return []
+		const start = this.keyIndexStart.at(keyIndex) ?? 0
+		const count = this.keyIndexCount.at(keyIndex) ?? 0
+		return Array.from(this.keyEntities.array.subarray(start, start + count))
+	}
+
+	/**
+	 * Create a unique composite index for a key=value pair.
+	 * Uses row-major indexing: `key * width + val`.
+	 */
+	kvToIndex(key: number, val: number) {
+		const width = this.stringTable.length
+		return key * width + val
+	}
+
+	/**
+	 * Get transferable objects for passing to another thread.
+	 */
+	transferables(): TagsTransferables {
+		return {
+			tagStart: this.tagStart.array.buffer,
+			tagCount: this.tagCount.array.buffer,
+			tagKeys: this.tagKeys.array.buffer,
+			tagVals: this.tagVals.array.buffer,
+			keyEntities: this.keyEntities.array.buffer,
+			keyIndexStart: this.keyIndexStart.array.buffer,
+			keyIndexCount: this.keyIndexCount.array.buffer,
+		}
+	}
+
+	/**
+	 * Get the approximate memory requirements for a given number of tags in bytes.
+	 */
+	static getBytesRequired(count: number) {
+		return (
+			count * Uint32Array.BYTES_PER_ELEMENT + // tagStart
+			count * Uint8Array.BYTES_PER_ELEMENT // tagCount
+		)
+	}
+
+	/**
+	 * Reconstruct a Tags index from transferable objects.
+	 */
+	static fromTransferables(
+		stringTable: StringTable,
+		transferables: TagsTransferables,
+	) {
+		const tagIndex = new Tags(stringTable, transferables)
+		tagIndex.indexBuilt = true
+		return tagIndex
+	}
+
+	/**
+	 * Update a ContentHasher with the tags data.
+	 * Hashes tag keys and values for each entity.
+	 */
+	updateHash(hasher: ContentHasher): ContentHasher {
+		return hasher
+			.update(this.tagStart.array)
+			.update(this.tagCount.array)
+			.update(this.tagKeys.array)
+			.update(this.tagVals.array)
+	}
 }