las-laz-js 0.3.0 → 0.4.0

package/src/write/writeLas.js

@@ -6,6 +6,15 @@
 // file handle) and patches the header totals on close. Both share
 // one header builder, so they produce identical bytes.
 //
+// With compress: true both write LAZ (point formats 0-3 and 6-8): the same
+// records, compressed chunkSize points at a time (LASzip's 50,000 by
+// default), after a LASzip VLR; the point data is an 8-byte offset to
+// the chunk table, the chunks, then the table — laid out as LAZperf
+// (PDAL) writes it. Chunks compress independently, so with threads
+// they compress on the LAZ worker pool (src/laz/pool.js), several to a
+// task, and are written strictly in chunk order whatever order they
+// finish in: the bytes are the same for any thread count.
+//
 // Ported from Kirra's LASWriter, with two changes: an invalid
 // version / format pair is REFUSED (Kirra silently downgraded the
 // format), and the coordinate system is written (GeoKey EPSG
@@ -16,9 +25,24 @@ import { LasError, ERROR_CODES } from "../errors.js";
 import { POINT_FORMATS } from "../format/PointFormats.js";
 import { chooseScaleOffset } from "./scaleOffset.js";
 import { encodeRecords } from "./encodeRecords.js";
+import { lazVlrFor, LASZIP_USER_ID, LASZIP_RECORD_ID } from "../laz/LazVlr.js";
+import { createChunkCompressor } from "../laz/compressChunk.js";
+import { encodeChunkTable } from "../laz/chunkTable.js";
+import { createLazPool, defaultThreads } from "../laz/pool.js";
 
 var LE = true;
 var HEADER_SIZE = { "1.0": 227, "1.1": 227, "1.2": 227, "1.3": 235, "1.4": 375 };
+var DEFAULT_CHUNK_SIZE = 50000;
+// 0xFFFFFFFF in the VLR's chunk size field means variable-size chunks
+var MAX_CHUNK_SIZE = 4294967294;
+// threads, automatic: a pool once a write has this many chunks (as openLas),
+// of at most AUTO_MAX_THREADS workers — gains flatten there, while the records
+// in flight grow with every worker (an explicit threads is honoured as given)
+var AUTO_MIN_CHUNKS = 4;
+var AUTO_MAX_THREADS = 4;
+// chunks per pool task, and tasks in flight per worker (one working, one queued)
+var CHUNKS_PER_TASK = 4;
+var TASKS_PER_WORKER = 2;
 
 function ascii(v, pos, text, max) {
 	for (var i = 0; i < max; i++) v.setUint8(pos + i, i < text.length ? text.charCodeAt(i) & 0x7F : 0); // bits-ok
@@ -68,7 +92,19 @@ function settle(points, opts) {
 	var minor = Number(version.split(".")[1]);
 	if (fmt >= 6 && minor < 4) throw new LasError(ERROR_CODES.INVALID_ARGUMENT, "point format " + fmt + " needs LAS 1.4 (asked for " + version + ")");
 	if ((fmt === 4 || fmt === 5) && minor < 3) throw new LasError(ERROR_CODES.INVALID_ARGUMENT, "point format " + fmt + " needs LAS 1.3 or later (asked for " + version + ")");
-	return { version: version, minor: minor, fmt: fmt, opts: opts };
+	return { version: version, minor: minor, fmt: fmt, opts: opts, laz: lazSetup(fmt, opts) };
+}
+
+// compress: the LASzip VLR and the chunk compressor. chunkSize is ignored without compress.
+function lazSetup(fmt, opts) {
+	if (!opts.compress) return null;
+	var chunkSize = opts.chunkSize == null ? DEFAULT_CHUNK_SIZE : opts.chunkSize;
+	if (!Number.isInteger(chunkSize) || chunkSize < 1 || chunkSize > MAX_CHUNK_SIZE) {
+		throw new LasError(ERROR_CODES.INVALID_ARGUMENT, "chunkSize must be a whole number of points from 1 to " + MAX_CHUNK_SIZE + " (got " + chunkSize + ")");
+	}
+	var len = POINT_FORMATS[fmt].size;
+	var made = lazVlrFor(fmt, len, chunkSize);   // UNSUPPORTED for formats 4, 5, 9, 10
+	return { chunkSize: chunkSize, vlr: made.vlr, vlrBytes: made.bytes, compress: createChunkCompressor(made.vlr, len) };
 }
 
 function freshTotals() {
@@ -80,6 +116,7 @@ function headerBytes(s, so, totals) {
 	var o = s.opts;
 	var headerSize = HEADER_SIZE[s.version];
 	var vlrs = crsRecords(o.crs, s.fmt);
+	if (s.laz) vlrs.push({ userId: LASZIP_USER_ID, recordId: LASZIP_RECORD_ID, description: "http://laszip.org", bytes: s.laz.vlrBytes });
 	var vlrSize = vlrs.reduce(function (n, r) { return n + 54 + r.bytes.length; }, 0);
 	var out = new Uint8Array(headerSize + vlrSize);
 	var v = new DataView(out.buffer);
@@ -101,7 +138,7 @@ function headerBytes(s, so, totals) {
 	v.setUint16(94, headerSize, LE);
 	v.setUint32(96, headerSize + vlrSize, LE);
 	v.setUint32(100, vlrs.length, LE);
-	v.setUint8(104, s.fmt);
+	v.setUint8(104, s.laz ? s.fmt + 128 : s.fmt);   // bit 7: compressed
 	v.setUint16(105, fmt.size, LE);
 	var legacyOk = s.fmt < 6 && totals.count < 4294967296;
 	v.setUint32(107, legacyOk ? totals.count : 0, LE);
@@ -144,9 +181,11 @@ function boundsOfPoints(points, origin) {
 }
 
 /**
- * Write a whole LAS file.
+ * Write a whole LAS file, or LAZ with compress: true (point formats 0-3 and 6-8).
  * @param {object} points - PointColumns (world coordinates, or relative to opts.localOrigin)
- * @param {object} [opts] - version, pointFormat, scale, offset, crs {epsg|wkt}, software, systemId, date, localOrigin, as ("bytes" | "blob")
+ * @param {object} [opts] - version, pointFormat, scale, offset, crs {epsg|wkt}, software, systemId, date, localOrigin, as ("bytes" | "blob"),
+ *   compress (write LAZ), chunkSize (points per LAZ chunk, default 50000; only with compress),
+ *   threads (LAZ compression workers: default automatic, from 4 chunks and at most 4 workers; 1 = this thread; n = n workers)
  * @returns {Promise<Uint8Array|Blob>}
  */
 export async function writeLas(points, opts) {
@@ -156,6 +195,10 @@ export async function writeLas(points, opts) {
 	if (o.scale && !o.offset) so = { scale: o.scale, offset: chooseScaleOffset(boundsOfPoints(points, o.localOrigin)).offset };
 	var len = POINT_FORMATS[s.fmt].size;
 	var totals = freshTotals();
+	if (s.laz) {
+		var parts = await lazParts(points, s, so, len, totals);
+		return o.as === "blob" ? new Blob(parts, { type: "application/octet-stream" }) : joinParts(parts);
+	}
 	// One buffer: records are encoded in place after the header, whose
 	// size does not depend on the totals; the header is filled in last
 	var headLength = headerBytes(s, so, totals).length;
@@ -166,6 +209,181 @@ export async function writeLas(points, opts) {
 	return o.as === "blob" ? new Blob([out], { type: "application/octet-stream" }) : out;
 }
 
+// Columns a..b of a PointColumns (views where the columns are typed arrays)
+function rangeOf(cols, a, b) {
+	if (a === 0 && b === cols.count) return cols;
+	var out = { count: b - a };
+	for (var k in cols) {
+		var c = cols[k];
+		if (k !== "count" && c && typeof c.length === "number") out[k] = typeof c.subarray === "function" ? c.subarray(a, b) : c.slice(a, b);
+	}
+	return out;
+}
+
+function joinParts(parts) {
+	var total = parts.reduce(function (n, p) { return n + p.length; }, 0);
+	var out = new Uint8Array(total), at = 0;
+	parts.forEach(function (p) { out.set(p, at); at += p.length; });
+	return out;
+}
+
+// The i64 offset to the chunk table (an absolute file position), as two u32 halves
+function tableOffsetBytes(tablePos) {
+	var b = new Uint8Array(8);
+	setU64(new DataView(b.buffer), 0, tablePos);
+	return b;
+}
+
+// threads as openLas: 0 = automatic
+function threadsOf(opts) { return opts.threads > 0 ? Math.floor(opts.threads) : 0; }
+
+// A pool of up to `most` workers (one per chunk at most, when the count is known), or null;
+// automatic: at most AUTO_MAX_THREADS
+async function startPool(opts, most) {
+	var n = Math.min(threadsOf(opts) || Math.min(AUTO_MAX_THREADS, await defaultThreads()), most);
+	if (n < 2) return null;
+	return opts._createPool ? opts._createPool(n) : createLazPool(n);
+}
+
+// The chunk loop both LAZ writers share. Records fill a buffer; each full
+// chunk is compressed on this thread, until the poolAt-th full chunk calls
+// makePool — from then on chunks go to the pool's workers CHUNKS_PER_TASK
+// at a time (or, with no pool, still to this thread). place(bytes) gets
+// each task's chunks, back to back, strictly in chunk order: a task that
+// finishes early waits for the ones before it. Records are encoded before
+// write()'s first await, so calls made without waiting keep their
+// order. write() returns once fewer than TASKS_PER_WORKER tasks per
+// worker are in flight (on this thread, once its chunks are placed), so
+// memory is bounded. Any error — on a worker, on this thread, in place()
+// or reported through fail(err) — stops the loop for good and closes the
+// pool, as finish() does once it has placed the last chunk; after
+// finish() is called, write() refuses more points (ABORTED).
+function lazChunks(s, so, len, totals, place, poolAt, makePool) {
+	var size = s.laz.chunkSize;
+	var buf = new Uint8Array(0), view = null, filled = 0, full = 0;
+	var pooled = null, closing = false, failure = null, ended = false;
+	var order = Promise.resolve(), inflight = [], chunks = [];
+
+	// grow the buffer to hold n records (doubling; one chunk on this thread, a task's on the pool)
+	function room(n) {
+		if (buf.length >= n * len) return;
+		var most = pooled ? CHUNKS_PER_TASK * size : size;
+		var grown = new Uint8Array(Math.max(n, Math.min(most, 2 * buf.length / len)) * len);
+		grown.set(buf.subarray(0, filled * len));
+		buf = grown;
+		view = new DataView(buf.buffer);
+	}
+	function shut() {
+		if (!pooled || closing) return;
+		closing = true;
+		pooled.then(function (pool) { if (pool) pool.close(); });
+	}
+	function stop(err) {
+		if (!failure) failure = err;
+		shut();
+	}
+	function refuse() {
+		if (failure) throw failure;
+		if (ended) throw new LasError(ERROR_CODES.ABORTED, "the LAZ writer is closed");
+	}
+	function onThisThread(records, counts) {
+		var parts = [], p = 0;
+		counts.forEach(function (n) { parts.push(s.laz.compress(records, p, n)); p += n * len; });
+		return { bytes: parts.length === 1 ? parts[0] : joinParts(parts), sizes: parts.map(function (c) { return c.length; }) };
+	}
+	function submit(records, counts) {
+		// on this thread now (its buffer is reused); on the pool from records the task then owns.
+		// An error on this thread rejects the task, as a worker's does, so it stops the loop below
+		var task;
+		if (!pooled) {
+			try { task = Promise.resolve(onThisThread(records, counts)); } catch (err) { task = Promise.reject(err); }
+		} else {
+			task = pooled.then(function (pool) {
+				return pool ? pool.compress({ vlr: s.laz.vlr, recordLength: len, records: records.buffer, counts: counts }) : onThisThread(records, counts);
+			});
+		}
+		task.catch(function () {});              // tasks after a failure are never waited for
+		var placed = order.then(function () { return task; }).then(function (r) {
+			r.sizes.forEach(function (n, i) { chunks.push({ pointCount: counts[i], byteCount: n }); });
+			return place(r.bytes);
+		});
+		placed.catch(stop);
+		order = placed;
+		inflight.push(placed);
+	}
+	// everything in the buffer as one task: full chunks, then a partial one at the end
+	function submitBuffer() {
+		var counts = [];
+		for (var k = 0; k < filled; k += size) counts.push(Math.min(size, filled - k));
+		var records = buf;
+		buf = new Uint8Array(0);
+		view = null;
+		filled = 0;
+		submit(records, counts);
+	}
+	function chunkFull() {
+		full++;
+		if (!pooled && full >= poolAt) pooled = makePool().catch(function () { return null; });
+		if (!pooled) { submit(buf, [size]); filled = 0; }
+		else if (filled === CHUNKS_PER_TASK * size) submitBuffer();
+	}
+	return {
+		// points, the first of which is point `first` of the caller's (for error messages)
+		write: async function (points, first) {
+			refuse();
+			try {
+				for (var a = 0; a < points.count;) {
+					var n = Math.min(size - filled % size, points.count - a);
+					room(filled + n);
+					encodeRecords(rangeOf(points, a, a + n), s.fmt, len, so, s.opts.localOrigin, view, filled * len, totals, first + a);
+					filled += n;
+					a += n;
+					totals.count += n;
+					if (filled % size === 0) chunkFull();
+				}
+			} catch (err) {
+				stop(err);
+				throw err;
+			}
+			var pool = pooled ? await pooled : null;
+			var limit = pool ? TASKS_PER_WORKER * pool.size : 0;
+			while (inflight.length && inflight.length >= limit) await inflight.shift();
+		},
+		finish: async function () {
+			refuse();
+			ended = true;
+			try {
+				if (filled) submitBuffer();
+				await order;
+			} catch (err) {
+				stop(err);
+				throw err;
+			} finally { shut(); }
+			return chunks;
+		},
+		fail: stop
+	};
+}
+
+// A whole LAZ file as parts: header + VLRs, table offset, chunks, chunk table.
+// The points go through the chunk loop a task's worth at a time, so only
+// the tasks in flight hold encoded records.
+async function lazParts(points, s, so, len, totals) {
+	var size = s.laz.chunkSize, count = Math.ceil(points.count / size), t = threadsOf(s.opts);
+	var parts = [null, null];
+	var poolAt = t === 1 || (!t && count < AUTO_MIN_CHUNKS) ? Infinity : 1;
+	var loop = lazChunks(s, so, len, totals, function (bytes) { parts.push(bytes); }, poolAt, function () { return startPool(s.opts, count); });
+	var step = CHUNKS_PER_TASK * size;
+	for (var a = 0; a < points.count; a += step) await loop.write(rangeOf(points, a, Math.min(points.count, a + step)), a);
+	var chunks = await loop.finish();
+	// the header's size does not depend on the totals
+	var tablePos = chunks.reduce(function (n, c) { return n + c.byteCount; }, headerBytes(s, so, totals).length + 8);
+	parts.push(encodeChunkTable(chunks, false));
+	parts[0] = headerBytes(s, so, totals);
+	parts[1] = tableOffsetBytes(tablePos);
+	return parts;
+}
+
 // One positioned write for any sink: an OPFS scratch writer (writeAt),
 // a Node FileHandle (write(buffer, offset, length, position), which may
 // write short), or a plain { write(bytes, position) }
@@ -184,11 +402,45 @@ function positionedWrite(sink) {
 	throw new LasError(ERROR_CODES.INVALID_ARGUMENT, "createLasWriter needs a sink with writeAt(bytes, position) or write(bytes, position)");
 }
 
+// The streaming LAZ writer: the chunk loop, placing chunks at the running
+// position. close() places the partial chunk, writes the chunk table after
+// it, then patches the table offset and the header totals. A failed write
+// there fails the loop too, so later calls keep rejecting (ruling R11); once
+// close() has begun, writePoints and close reject (ABORTED).
+async function createLazWriter(s, so, len, totals, put, start) {
+	var pos = start + 8, t = threadsOf(s.opts);
+	// until close(), the table offset is LASzip's "not written yet" (-1)
+	await put(joinParts([headerBytes(s, so, totals), new Uint8Array(8).fill(255)]), 0);
+	var poolAt = t === 1 ? Infinity : t ? 1 : AUTO_MIN_CHUNKS;
+	var loop = lazChunks(s, so, len, totals, function (bytes) {
+		var at = pos;
+		pos += bytes.length;
+		return put(bytes, at);
+	}, poolAt, function () { return startPool(s.opts, Infinity); });
+	return {
+		writePoints: function (points) { return loop.write(points, 0); },
+		close: async function () {
+			var chunks = await loop.finish();
+			var tablePos = pos;
+			try {
+				await put(encodeChunkTable(chunks, false), tablePos);
+				await put(tableOffsetBytes(tablePos), start);
+				await put(headerBytes(s, so, totals), 0);
+			} catch (err) {
+				loop.fail(err);
+				throw err;
+			}
+			return { count: totals.count };
+		}
+	};
+}
+
 /**
- * Stream a LAS file to a sink. Header totals are patched on close().
+ * Stream a LAS (or, with compress: true, LAZ) file to a sink. Header totals are patched on close().
  * @param {object} sink - an OPFS scratch writer (openScratchSyncWriter), a Node
  *   FileHandle, or any { write(bytes, position) }
- * @param {object} opts - as writeLas; scale and offset are required (the bounds are not known yet)
+ * @param {object} opts - as writeLas; scale and offset are required (the bounds are not known yet).
+ *   With compress and threads automatic, the pool (at most 4 workers) starts at the 4th full chunk
  * @returns {Promise<{writePoints: function(object): Promise, close: function(): Promise<{count:number}>}>}
  */
 export async function createLasWriter(sink, opts) {
@@ -200,6 +452,7 @@ export async function createLasWriter(sink, opts) {
 	var totals = freshTotals();
 	var put = positionedWrite(sink);
 	var start = headerBytes(s, so, totals).length;
+	if (s.laz) return createLazWriter(s, so, len, totals, put, start);
 	var pending = [];
 	await put(headerBytes(s, so, totals), 0);
 	return {