las-laz-js 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +17 -4
- package/package.json +1 -1
- package/src/LasReader.js +70 -23
- package/src/index.js +1 -1
- package/src/laz/ArithmeticDecoder.js +3 -3
- package/src/laz/IntegerDecompressor.js +7 -3
- package/src/laz/chunkWorker.js +53 -0
- package/src/laz/pool.js +151 -0
- package/src/laz/v2/point10.js +9 -6
- package/src/laz/v3/layered.js +9 -8
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.1 (2026-10-21)
|
|
4
|
+
|
|
5
|
+
Worker fixes.
|
|
6
|
+
|
|
7
|
+
- Node: a script that stopped part way through a parallel LAZ read (one `chunks()` window, then
|
|
8
|
+
done) never exited, because an idle pool kept Node running. Workers now hold Node only while
|
|
9
|
+
they have work.
|
|
10
|
+
- Node: `node --input-type=module -e` scripts can decode in parallel. Workers used to inherit
|
|
11
|
+
`--input-type` and fail to start.
|
|
12
|
+
- A worker that cannot start (blocked or failing script) now means one-thread decoding, as
|
|
13
|
+
documented, rather than a failed read. The pool waits for every worker to report ready.
|
|
14
|
+
|
|
15
|
+
## 0.3.0 (2026-10-21)
|
|
16
|
+
|
|
17
|
+
Faster LAZ.
|
|
18
|
+
|
|
19
|
+
- LAZ chunks decode in parallel on a pool of workers (browser Workers, Node worker_threads; one
|
|
20
|
+
fewer than the cores, at most 16), with the next window decoding while the current one becomes
|
|
21
|
+
columns. A 10-million-point LAZ reads in about 0.6 s (3.9 s on one thread); a 1.2 GB,
|
|
22
|
+
160-million-point LAZ in about 12 s (102 s in 0.2.0). Output is identical.
|
|
23
|
+
- `openLas(src, { threads })`: automatic by default (used from 4 chunks up); `threads: 1` decodes
|
|
24
|
+
on the calling thread. Falls back to one thread when workers cannot start.
|
|
25
|
+
- The decoder's hot loop does less: about 30% faster on one thread, same results.
|
|
26
|
+
- Demo: decodes in the background with a progress bar, then shows the cloud once; colour, ramp
|
|
27
|
+
and range recolour the points already read instead of reading the file again.
|
|
28
|
+
|
|
3
29
|
## 0.2.0 (2026-10-21)
|
|
4
30
|
|
|
5
31
|
Reads LAZ.
|
package/README.md
CHANGED
|
@@ -12,8 +12,8 @@ runs in the browser and in Node — and built for files far larger than memory.
|
|
|
12
12
|
- LAZ: point formats 0–3 and 6–8, with extra bytes, read exactly like LAS — the same calls, the
|
|
13
13
|
same options, decoded chunk by chunk in plain JavaScript (no WebAssembly).
|
|
14
14
|
- Big files: a 1.1 GB, 34-million-point LAS reads in under a second in Node, and a 10-million-point
|
|
15
|
-
LAZ decodes
|
|
16
|
-
for the one exception).
|
|
15
|
+
LAZ decodes on every core (a 10-million-point file in about 0.6 s); neither is held in memory
|
|
16
|
+
whole (see LAZ below for the one exception).
|
|
17
17
|
- Also: a ready-made module Worker (`las-laz-js/worker`) and a
|
|
18
18
|
[Three.js demo](https://brentbuffham.github.io/las-laz-js/).
|
|
19
19
|
- Not yet: writing LAZ (0.3.0).
|
|
@@ -30,8 +30,9 @@ Node 22 or newer, or any current browser. No dependencies.
|
|
|
30
30
|
|
|
31
31
|
Try it in the browser: **https://brentbuffham.github.io/las-laz-js/**
|
|
32
32
|
|
|
33
|
-
Drop a LAS or LAZ file onto the page. It is read locally (never uploaded), in the
|
|
34
|
-
Worker
|
|
33
|
+
Drop a LAS or LAZ file onto the page. It is read locally (never uploaded), decoded in the
|
|
34
|
+
library's Worker (LAZ on every core) behind a progress bar, then shown once in a Three.js viewer;
|
|
35
|
+
colour changes recolour the points already read, without reading the file again. The side panel shows the header, VLRs, coordinate
|
|
35
36
|
system and LAZ compression; reads with class, return, bounds, thinning and dedupe filters;
|
|
36
37
|
colours by RGB, intensity, elevation or class; lists point statistics; and saves the points back
|
|
37
38
|
out as LAS. To run it locally: `npm run demo`, then open http://localhost:8080/.
|
|
@@ -89,6 +90,13 @@ reading the LAS it was compressed from.
|
|
|
89
90
|
naming what it found — never garbage points.
|
|
90
91
|
- A LAZ file cut short is `TRUNCATED`.
|
|
91
92
|
|
|
93
|
+
**Speed.** LAZ chunks are independent, so they are decoded on a pool of workers — browser
|
|
94
|
+
`Worker`s or Node `worker_threads`, one fewer than the cores (at most 16) — while the next window
|
|
95
|
+
is already decoding. A 10-million-point LAZ reads in about 0.6 s on a 24-core machine (about 4 s
|
|
96
|
+
on one core). Files under four chunks (200,000 points) stay on one thread, where starting workers
|
|
97
|
+
would cost more than it saves. `openLas(file, { threads: 1 })` decodes on the calling thread;
|
|
98
|
+
`threads: n` asks for n workers. If workers cannot start, it falls back to one thread.
|
|
99
|
+
|
|
92
100
|
## Files bigger than memory
|
|
93
101
|
|
|
94
102
|
`openLas` reads only the header and the VLRs. Points are read in windows (64 MiB of records by
|
|
@@ -152,6 +160,11 @@ worker.postMessage({ id: 2, type: "chunks", payload: { fields: ["x", "y", "z"],
|
|
|
152
160
|
The protocol is documented in `src/workers/workerProtocol.js` and can be driven directly with
|
|
153
161
|
`createWorkerHandler(post)`.
|
|
154
162
|
|
|
163
|
+
**Bundlers.** The worker entry and the LAZ decode workers are found through
|
|
164
|
+
`new Worker(new URL(..., import.meta.url))`, which Vite and webpack 5 follow. With Vite, keep the
|
|
165
|
+
package out of dependency pre-bundling (`optimizeDeps: { exclude: ["las-laz-js"] }`), so the
|
|
166
|
+
worker files stay next to the module that loads them.
|
|
167
|
+
|
|
155
168
|
## Errors
|
|
156
169
|
|
|
157
170
|
Every failure is a `LasError` with a stable `code` (`ERROR_CODES`) and, for byte-level problems,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "las-laz-js",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Zero-dependency JavaScript reader and writer for ASPRS LAS point clouds, and reader for LAZ (LASzip), for the browser and Node, built for files larger than memory.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
package/src/LasReader.js
CHANGED
|
@@ -22,11 +22,13 @@ import { windows, lazWindows } from "./read/windows.js";
|
|
|
22
22
|
import { findLazVlr } from "./laz/LazVlr.js";
|
|
23
23
|
import { readChunks } from "./laz/chunkTable.js";
|
|
24
24
|
import { createChunkDecoder } from "./laz/decompressChunk.js";
|
|
25
|
+
import { createLazPool, defaultThreads } from "./laz/pool.js";
|
|
25
26
|
import { makeKeep, countKept } from "./filter/filters.js";
|
|
26
27
|
import { dedupe } from "./filter/dedupe.js";
|
|
27
28
|
import { openScratchSyncWriter } from "./io/ScratchBinaryFile.js";
|
|
28
29
|
|
|
29
30
|
var DEFAULT_WINDOW = 64 * 1024 * 1024;
|
|
31
|
+
var AUTO_MIN_CHUNKS = 4;
|
|
30
32
|
|
|
31
33
|
function memorySource(bytes) {
|
|
32
34
|
return { size: bytes.length, read: async function (a, b) {
|
|
@@ -62,7 +64,8 @@ async function toSource(source) {
|
|
|
62
64
|
/**
|
|
63
65
|
* Open a LAS (or LAZ) file and read its header and VLRs.
|
|
64
66
|
* @param {*} source
|
|
65
|
-
* @param {{windowSize?: number}} [opts]
|
|
67
|
+
* @param {{windowSize?: number, threads?: number}} [opts] - threads: LAZ decode workers
|
|
68
|
+
* (default automatic: cores - 1, at most 16; 1 = decode on this thread)
|
|
66
69
|
* @returns {Promise<LasReader>}
|
|
67
70
|
*/
|
|
68
71
|
export async function openLas(source, opts) {
|
|
@@ -79,7 +82,9 @@ export async function openLas(source, opts) {
|
|
|
79
82
|
if (header.numberOfEvlrs > 0 && header.evlrStart > 0 && header.evlrStart < src.size) {
|
|
80
83
|
evlrs = await walkEvlrs(src, header.evlrStart, header.numberOfEvlrs);
|
|
81
84
|
}
|
|
82
|
-
|
|
85
|
+
var reader = new LasReader(src, header, vlrs, evlrs, (opts && opts.windowSize) || DEFAULT_WINDOW);
|
|
86
|
+
reader._threads = opts && opts.threads > 0 ? Math.floor(opts.threads) : 0; // 0 = automatic
|
|
87
|
+
return reader;
|
|
83
88
|
}
|
|
84
89
|
|
|
85
90
|
// EVLRs sit after the points and can be huge (waveform data): read each
|
|
@@ -153,7 +158,19 @@ LasReader.prototype._plan = async function () {
|
|
|
153
158
|
var view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
154
159
|
win.chunks.forEach(function (c) { decode(packed, c.start - win.start, c.end - win.start, c.count, view, (c.firstRecord - win.firstRecord) * len, win.start); });
|
|
155
160
|
return buf.subarray(0, win.count * len);
|
|
156
|
-
}
|
|
161
|
+
},
|
|
162
|
+
// The same records, each chunk decoded by a pool worker from its own copy of the bytes
|
|
163
|
+
parallel: async function (win, pool) {
|
|
164
|
+
var packed = await src.read(win.start, win.end);
|
|
165
|
+
var out = new Uint8Array(win.count * len);
|
|
166
|
+
await Promise.all(win.chunks.map(function (c) {
|
|
167
|
+
var piece = packed.slice(c.start - win.start, c.end - win.start);
|
|
168
|
+
return pool.decode({ vlr: vlr, recordLength: len, packed: piece.buffer, chunks: [{ s: 0, e: piece.length, count: c.count }], base: c.start })
|
|
169
|
+
.then(function (rec) { out.set(rec, (c.firstRecord - win.firstRecord) * len); });
|
|
170
|
+
}));
|
|
171
|
+
return out;
|
|
172
|
+
},
|
|
173
|
+
chunkCount: lazList.reduce(function (n, w) { return n + w.chunks.length; }, 0)
|
|
157
174
|
};
|
|
158
175
|
}
|
|
159
176
|
this._planned = plan;
|
|
@@ -179,32 +196,62 @@ LasReader.prototype._scan = async function* (opts, into) {
|
|
|
179
196
|
var keep = makeKeep(opts, h);
|
|
180
197
|
var fields = resolveFields(h, opts.fields);
|
|
181
198
|
var list = plan.list;
|
|
182
|
-
var
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
199
|
+
var pool = await this._pool(plan);
|
|
200
|
+
try {
|
|
201
|
+
var buf = !pool && plan.bufferSize ? new Uint8Array(plan.bufferSize) : null;
|
|
202
|
+
var thin = null;
|
|
203
|
+
if (opts.maxPoints > 0) {
|
|
204
|
+
// Thin the points that pass the filters; with filters set, count them first (a second read)
|
|
205
|
+
var passing = keep ? await this._countPassing(plan, pool, buf, keep, opts.signal) : h.pointCount;
|
|
206
|
+
if (passing > opts.maxPoints) thin = { total: passing, target: opts.maxPoints, acc: 0 };
|
|
207
|
+
}
|
|
208
|
+
var kept = 0, w = 0;
|
|
209
|
+
for await (var bytes of windowBytes(plan, pool, buf, opts.signal)) {
|
|
210
|
+
var win = list[w];
|
|
211
|
+
var cols = decodeRecords(bytes, h, { fields: fields, keep: keep, thin: thin, localOrigin: opts.localOrigin, float32: opts.float32, into: into ? into(win) : null });
|
|
212
|
+
kept += cols.count;
|
|
213
|
+
w++;
|
|
214
|
+
if (typeof opts.onProgress === "function") opts.onProgress(w / list.length, "read " + kept + " points");
|
|
215
|
+
yield cols;
|
|
216
|
+
}
|
|
191
217
|
if (opts.signal && opts.signal.aborted) throw new LasError(ERROR_CODES.ABORTED, "read aborted");
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
var cols = decodeRecords(bytes, h, { fields: fields, keep: keep, thin: thin, localOrigin: opts.localOrigin, float32: opts.float32, into: into ? into(win) : null });
|
|
195
|
-
kept += cols.count;
|
|
196
|
-
if (typeof opts.onProgress === "function") opts.onProgress((w + 1) / list.length, "read " + kept + " points");
|
|
197
|
-
yield cols;
|
|
218
|
+
} finally {
|
|
219
|
+
if (pool) pool.close();
|
|
198
220
|
}
|
|
199
|
-
if (opts.signal && opts.signal.aborted) throw new LasError(ERROR_CODES.ABORTED, "read aborted");
|
|
200
221
|
};
|
|
201
222
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
223
|
+
// Each window's raw records in order. With a pool the next window is already
|
|
224
|
+
// decoding while this one is used; without one, one buffer is reused.
|
|
225
|
+
async function* windowBytes(plan, pool, buf, signal) {
|
|
226
|
+
var list = plan.list;
|
|
227
|
+
function fetch(i) {
|
|
228
|
+
var p = plan.parallel(list[i], pool);
|
|
229
|
+
p.catch(function () {}); // a prefetch abandoned by an abort must not go unhandled
|
|
230
|
+
return p;
|
|
231
|
+
}
|
|
232
|
+
var next = pool && list.length ? fetch(0) : null;
|
|
233
|
+
for (var w = 0; w < list.length; w++) {
|
|
205
234
|
if (signal && signal.aborted) throw new LasError(ERROR_CODES.ABORTED, "read aborted");
|
|
206
|
-
|
|
235
|
+
if (!pool) { yield await plan.records(list[w], buf); continue; }
|
|
236
|
+
var current = next;
|
|
237
|
+
next = w + 1 < list.length ? fetch(w + 1) : null;
|
|
238
|
+
yield await current;
|
|
207
239
|
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// A decode pool for this scan, or null: LAS, threads: 1, one chunk, or no workers available
|
|
243
|
+
LasReader.prototype._pool = async function (plan) {
|
|
244
|
+
if (!plan.parallel || this._threads === 1) return null;
|
|
245
|
+
// automatic: only when there is enough to share (starting workers costs about as much as
|
|
246
|
+
// decoding a few chunks); an explicit thread count is always honoured
|
|
247
|
+
if (!this._threads && plan.chunkCount < AUTO_MIN_CHUNKS) return null;
|
|
248
|
+
var n = Math.min(this._threads || await defaultThreads(), plan.chunkCount);
|
|
249
|
+
return n >= 2 ? createLazPool(n) : null;
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
LasReader.prototype._countPassing = async function (plan, pool, buf, keep, signal) {
|
|
253
|
+
var n = 0;
|
|
254
|
+
for await (var bytes of windowBytes(plan, pool, buf, signal)) n += countKept(bytes, this.header.recordLength, keep);
|
|
208
255
|
return n;
|
|
209
256
|
};
|
|
210
257
|
|
package/src/index.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
//==========================================================
|
|
7
7
|
|
|
8
8
|
/** Library version, matches package.json. */
|
|
9
|
-
export var VERSION = "0.
|
|
9
|
+
export var VERSION = "0.3.1";
|
|
10
10
|
|
|
11
11
|
export { LasError, ERROR_CODES } from "./errors.js";
|
|
12
12
|
export { openLas, LasReader } from "./LasReader.js";
|
|
@@ -77,7 +77,7 @@ ArithmeticDecoder.prototype.decodeSymbol = function (m) {
|
|
|
77
77
|
if (m.decoderTable) {
|
|
78
78
|
this.range = this.range >>> 15; // DM_LENGTH_SHIFT
|
|
79
79
|
if (this.range === 0) throw new LasError(ERROR_CODES.TRUNCATED, "corrupt LAZ stream (zero interval)", this.base + this.p);
|
|
80
|
-
var dv =
|
|
80
|
+
var dv = (this.value / this.range) >>> 0; // both u32: truncation = floor
|
|
81
81
|
var t = dv >>> m.tableShift;
|
|
82
82
|
sym = m.decoderTable[t];
|
|
83
83
|
n = m.decoderTable[t + 1] + 1;
|
|
@@ -114,7 +114,7 @@ ArithmeticDecoder.prototype.readBits = function (bits) {
|
|
|
114
114
|
return this.readBits(bits - 16) * 65536 + low;
|
|
115
115
|
}
|
|
116
116
|
this.range = this.range >>> bits;
|
|
117
|
-
var sym =
|
|
117
|
+
var sym = (this.value / this.range) >>> 0;
|
|
118
118
|
this.value -= this.range * sym;
|
|
119
119
|
if (this.range < AC_MIN_LENGTH) this._renorm();
|
|
120
120
|
return sym;
|
|
@@ -123,7 +123,7 @@ ArithmeticDecoder.prototype.readBits = function (bits) {
|
|
|
123
123
|
/** 16 raw bits. */
|
|
124
124
|
ArithmeticDecoder.prototype.readShort = function () {
|
|
125
125
|
this.range = this.range >>> 16;
|
|
126
|
-
var sym =
|
|
126
|
+
var sym = (this.value / this.range) >>> 0;
|
|
127
127
|
this.value -= this.range * sym;
|
|
128
128
|
if (this.range < AC_MIN_LENGTH) this._renorm();
|
|
129
129
|
return sym;
|
|
@@ -15,6 +15,10 @@
|
|
|
15
15
|
|
|
16
16
|
import { ArithmeticModel, ArithmeticBitModel } from "./models.js";
|
|
17
17
|
|
|
18
|
+
// 2^0 .. 2^32, exact in Numbers (the hot loop needs them per integer)
|
|
19
|
+
var POW2 = new Float64Array(33);
|
|
20
|
+
for (var p2 = 0; p2 <= 32; p2++) POW2[p2] = Math.pow(2, p2);
|
|
21
|
+
|
|
18
22
|
/**
|
|
19
23
|
* @param {number} [bits] - integer width (default 16; 32 wraps as i32)
|
|
20
24
|
* @param {number} [contexts] - number of k models (default 1)
|
|
@@ -60,10 +64,10 @@ IntegerDecompressor.prototype.decompress = function (dec, pred, context) {
|
|
|
60
64
|
c = dec.decodeSymbol(this.mCorrector[k - 1]);
|
|
61
65
|
} else {
|
|
62
66
|
var k1 = k - this.bitsHigh;
|
|
63
|
-
c = dec.decodeSymbol(this.mCorrector[k - 1]) *
|
|
67
|
+
c = dec.decodeSymbol(this.mCorrector[k - 1]) * POW2[k1] + dec.readBits(k1);
|
|
64
68
|
}
|
|
65
|
-
if (c >=
|
|
66
|
-
else c -=
|
|
69
|
+
if (c >= POW2[k - 1]) c += 1;
|
|
70
|
+
else c -= POW2[k] - 1;
|
|
67
71
|
} else {
|
|
68
72
|
c = this.corrMin;
|
|
69
73
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// src/laz/chunkWorker.js
|
|
2
|
+
//==========================================================
|
|
3
|
+
// Worker entry for parallel LAZ decoding (src/laz/pool.js): decodes
|
|
4
|
+
// whole chunks into raw LAS records and sends them back (transferred).
|
|
5
|
+
// Runs as a browser module Worker or a Node worker_threads Worker.
|
|
6
|
+
// In: { id, vlr, recordLength, packed: ArrayBuffer, chunks: [{ s, e, count }], base }
|
|
7
|
+
// (s / e index into packed; base is packed's file offset)
|
|
8
|
+
// Out: { ready: true } once listening, then per task
|
|
9
|
+
// { id, records: ArrayBuffer } or { id, error: { code, message, offset } }
|
|
10
|
+
//
|
|
11
|
+
// Uses the chunk decoder translated from laz-rs (https://github.com/laz-rs/laz-rs),
|
|
12
|
+
// itself a port of LASzip by rapidlasso. (c) 2007-2015 martin isenburg,
|
|
13
|
+
// rapidlasso; (c) 2019-2023 Thomas Montaigu.
|
|
14
|
+
// Licensed under the Apache License, Version 2.0 (see LICENSE-APACHE and
|
|
15
|
+
// NOTICE). Changes: written in JavaScript as a worker entry, Brent Buffham 2026.
|
|
16
|
+
//==========================================================
|
|
17
|
+
|
|
18
|
+
import { createChunkDecoder } from "./decompressChunk.js";
|
|
19
|
+
|
|
20
|
+
// node:worker_threads is named at run time so browser bundlers leave it alone
|
|
21
|
+
var NODE_WORKER_THREADS = ["node", "worker_threads"].join(":");
|
|
22
|
+
|
|
23
|
+
var port = null;
|
|
24
|
+
if (typeof self !== "undefined" && typeof self.postMessage === "function") {
|
|
25
|
+
port = { post: function (m, t) { self.postMessage(m, t); } };
|
|
26
|
+
self.onmessage = function (e) { handle(e.data); };
|
|
27
|
+
} else {
|
|
28
|
+
var wt = await import(/* webpackIgnore: true */ /* @vite-ignore */ NODE_WORKER_THREADS);
|
|
29
|
+
port = { post: function (m, t) { wt.parentPort.postMessage(m, t); } };
|
|
30
|
+
wt.parentPort.on("message", handle);
|
|
31
|
+
}
|
|
32
|
+
port.post({ ready: true }); // the pool waits for this before sending work
|
|
33
|
+
|
|
34
|
+
var cached = { key: null, decode: null };
|
|
35
|
+
|
|
36
|
+
function handle(m) {
|
|
37
|
+
try {
|
|
38
|
+
var key = JSON.stringify(m.vlr) + "/" + m.recordLength;
|
|
39
|
+
if (cached.key !== key) cached = { key: key, decode: createChunkDecoder(m.vlr, m.recordLength) };
|
|
40
|
+
var bytes = new Uint8Array(m.packed);
|
|
41
|
+
var total = m.chunks.reduce(function (s, c) { return s + c.count; }, 0);
|
|
42
|
+
var out = new ArrayBuffer(total * m.recordLength);
|
|
43
|
+
var view = new DataView(out), o = 0;
|
|
44
|
+
for (var i = 0; i < m.chunks.length; i++) {
|
|
45
|
+
var c = m.chunks[i];
|
|
46
|
+
cached.decode(bytes, c.s, c.e, c.count, view, o, m.base);
|
|
47
|
+
o += c.count * m.recordLength;
|
|
48
|
+
}
|
|
49
|
+
port.post({ id: m.id, records: out }, [out]);
|
|
50
|
+
} catch (err) {
|
|
51
|
+
port.post({ id: m.id, error: { code: err && err.code, message: err && err.message ? err.message : String(err), offset: err && err.offset } });
|
|
52
|
+
}
|
|
53
|
+
}
|
package/src/laz/pool.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
// src/laz/pool.js
|
|
2
|
+
//==========================================================
|
|
3
|
+
// A pool of workers that decode LAZ chunks in parallel (chunks are
|
|
4
|
+
// independent, so N cores decode about N times as fast). Browser module
|
|
5
|
+
// Workers or Node worker_threads, running src/laz/chunkWorker.js. Tasks
|
|
6
|
+
// queue and go to the next free worker; results come back as transferred
|
|
7
|
+
// buffers. close() terminates every worker and fails what is still
|
|
8
|
+
// queued, so nothing outlives a read. When workers cannot start (no
|
|
9
|
+
// Worker, a blocked or failing script), createLazPool returns null and
|
|
10
|
+
// the reader decodes on its own thread.
|
|
11
|
+
//
|
|
12
|
+
// Uses the chunk decoder translated from laz-rs (https://github.com/laz-rs/laz-rs),
|
|
13
|
+
// itself a port of LASzip by rapidlasso. (c) 2007-2015 martin isenburg,
|
|
14
|
+
// rapidlasso; (c) 2019-2023 Thomas Montaigu.
|
|
15
|
+
// Licensed under the Apache License, Version 2.0 (see LICENSE-APACHE and
|
|
16
|
+
// NOTICE). Changes: written in JavaScript as a worker pool, Brent Buffham 2026.
|
|
17
|
+
//==========================================================
|
|
18
|
+
|
|
19
|
+
import { LasError, ERROR_CODES } from "../errors.js";
|
|
20
|
+
|
|
21
|
+
var NODE_WORKER_THREADS = ["node", "worker_threads"].join(":");
|
|
22
|
+
var NODE_OS = ["node", "os"].join(":");
|
|
23
|
+
var MAX_THREADS = 16;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* How many decode workers to use by default: one fewer than the cores, at most 16.
|
|
27
|
+
* @returns {Promise<number>}
|
|
28
|
+
*/
|
|
29
|
+
export async function defaultThreads() {
|
|
30
|
+
var cores = 1;
|
|
31
|
+
if (typeof navigator !== "undefined" && navigator.hardwareConcurrency) cores = navigator.hardwareConcurrency;
|
|
32
|
+
else {
|
|
33
|
+
try {
|
|
34
|
+
var os = await import(/* webpackIgnore: true */ /* @vite-ignore */ NODE_OS);
|
|
35
|
+
cores = typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length;
|
|
36
|
+
} catch (e) { cores = 1; }
|
|
37
|
+
}
|
|
38
|
+
return Math.max(1, Math.min(MAX_THREADS, cores - 1));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Node worker options: the parent's flags are inherited, except --input-type
|
|
42
|
+
// (node --input-type=module -e passes it on, and a worker has no string input
|
|
43
|
+
// for it). Only then is execArgv given: an explicit one is checked flag by
|
|
44
|
+
// flag, and some parents (the test runner) hold flags a worker may not take.
|
|
45
|
+
function nodeWorkerOptions(argv) {
|
|
46
|
+
var isInputType = function (a, i) { return /^--input-type(=|$)/.test(a) || argv[i - 1] === "--input-type"; };
|
|
47
|
+
if (!argv.some(isInputType)) return undefined;
|
|
48
|
+
return { execArgv: argv.filter(function (a, i) { return !isInputType(a, i); }) };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// One worker, browser or Node, behind the same small interface. Resolves once
|
|
52
|
+
// the worker says it is ready; rejects if it fails or exits first (a blocked
|
|
53
|
+
// or broken script), so a pool that cannot start is known before any task.
|
|
54
|
+
async function spawn(onMessage, onFail, nodeUrl) {
|
|
55
|
+
var started = false, ready, failed;
|
|
56
|
+
var up = new Promise(function (res, rej) { ready = res; failed = rej; });
|
|
57
|
+
function message(m) { if (m && m.ready) { started = true; ready(); } else onMessage(m); }
|
|
58
|
+
function error(e) { if (started) onFail(e); else failed(e); }
|
|
59
|
+
var handle;
|
|
60
|
+
if (typeof Worker === "function") {
|
|
61
|
+
// written inline so bundlers (Vite, webpack 5) see the worker and emit it
|
|
62
|
+
var w = new Worker(new URL("./chunkWorker.js", import.meta.url), { type: "module" });
|
|
63
|
+
w.onmessage = function (e) { message(e.data); };
|
|
64
|
+
w.onerror = function (e) { if (e && e.preventDefault) e.preventDefault(); error(new Error(e && e.message ? e.message : "LAZ worker failed")); };
|
|
65
|
+
handle = { post: function (m, t) { w.postMessage(m, t); }, close: function () { w.terminate(); } };
|
|
66
|
+
} else {
|
|
67
|
+
var wt = await import(/* webpackIgnore: true */ /* @vite-ignore */ NODE_WORKER_THREADS);
|
|
68
|
+
var nw = new wt.Worker(nodeUrl || new URL("./chunkWorker.js", import.meta.url), nodeWorkerOptions(process.execArgv));
|
|
69
|
+
nw.on("message", message);
|
|
70
|
+
nw.on("error", error);
|
|
71
|
+
nw.on("exit", function (code) { if (!started) failed(new Error("LAZ worker exited (" + code + ") before it started")); });
|
|
72
|
+
// Node runs while a worker has a task and not while it is idle, so a forgotten
|
|
73
|
+
// pool never keeps it running (unref after the "message" listener, which refs again)
|
|
74
|
+
handle = { post: function (m, t) { nw.postMessage(m, t); }, close: function () { return nw.terminate(); }, hold: function () { nw.ref(); }, release: function () { nw.unref(); } };
|
|
75
|
+
}
|
|
76
|
+
try { await up; } catch (e) { handle.close(); throw e; }
|
|
77
|
+
if (handle.release) handle.release();
|
|
78
|
+
return handle;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* @param {number} n - workers to start (2 or more)
|
|
83
|
+
* @param {URL} [nodeUrl] - a Node worker entry in place of chunkWorker.js (tests)
|
|
84
|
+
* @returns {Promise<{size:number, decode:function(object): Promise<Uint8Array>, close:function(): void}|null>}
|
|
85
|
+
*/
|
|
86
|
+
export async function createLazPool(n, nodeUrl) {
|
|
87
|
+
var workers = [], idle = [], queue = [], waiting = new Map(), nextId = 1, closed = false;
|
|
88
|
+
|
|
89
|
+
function fail(err) {
|
|
90
|
+
// a worker that dies takes the pool with it: every waiting task fails
|
|
91
|
+
var e = err instanceof LasError ? err : new LasError(ERROR_CODES.UNSUPPORTED, "a LAZ decode worker failed: " + (err && err.message ? err.message : String(err)));
|
|
92
|
+
waiting.forEach(function (job) { job.reject(e); });
|
|
93
|
+
waiting.clear();
|
|
94
|
+
queue.forEach(function (job) { job.reject(e); });
|
|
95
|
+
queue = [];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function pump() {
|
|
99
|
+
while (idle.length && queue.length) {
|
|
100
|
+
var w = idle.pop(), job = queue.shift();
|
|
101
|
+
job.worker = w;
|
|
102
|
+
waiting.set(job.msg.id, job);
|
|
103
|
+
if (w.hold) w.hold();
|
|
104
|
+
w.post(job.msg, [job.msg.packed]);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function onMessage(m) {
|
|
109
|
+
var job = waiting.get(m.id);
|
|
110
|
+
if (!job) return;
|
|
111
|
+
waiting.delete(m.id);
|
|
112
|
+
if (job.worker.release) job.worker.release();
|
|
113
|
+
idle.push(job.worker);
|
|
114
|
+
if (m.error) job.reject(new LasError(m.error.code || ERROR_CODES.UNSUPPORTED, m.error.message, m.error.offset));
|
|
115
|
+
else job.resolve(new Uint8Array(m.records));
|
|
116
|
+
pump();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
var starts = [];
|
|
120
|
+
for (var i = 0; i < n; i++) starts.push(spawn(onMessage, fail, nodeUrl));
|
|
121
|
+
var settled = await Promise.allSettled(starts);
|
|
122
|
+
workers = settled.filter(function (s) { return s.status === "fulfilled"; }).map(function (s) { return s.value; });
|
|
123
|
+
if (workers.length < n) {
|
|
124
|
+
// any worker that cannot start means the reader decodes on its own thread
|
|
125
|
+
workers.forEach(function (w) { w.close(); });
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
idle = workers.slice();
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
size: n,
|
|
132
|
+
/**
|
|
133
|
+
* Decode chunks from their own copy of the compressed bytes.
|
|
134
|
+
* @param {{vlr:object, recordLength:number, packed:ArrayBuffer, chunks:Array, base:number}} task
|
|
135
|
+
* @returns {Promise<Uint8Array>} the chunks' raw LAS records, in order
|
|
136
|
+
*/
|
|
137
|
+
decode: function (task) {
|
|
138
|
+
if (closed) return Promise.reject(new LasError(ERROR_CODES.ABORTED, "the LAZ decode pool is closed"));
|
|
139
|
+
return new Promise(function (resolve, reject) {
|
|
140
|
+
queue.push({ msg: Object.assign({ id: nextId++ }, task), resolve: resolve, reject: reject });
|
|
141
|
+
pump();
|
|
142
|
+
});
|
|
143
|
+
},
|
|
144
|
+
close: function () {
|
|
145
|
+
if (closed) return;
|
|
146
|
+
closed = true;
|
|
147
|
+
fail(new LasError(ERROR_CODES.ABORTED, "the read stopped"));
|
|
148
|
+
workers.forEach(function (w) { w.close(); });
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
}
|
package/src/laz/v2/point10.js
CHANGED
|
@@ -61,6 +61,9 @@ StreamingMedian.prototype.add = function (x) {
|
|
|
61
61
|
};
|
|
62
62
|
|
|
63
63
|
function models(n, symbols) { var out = []; for (var i = 0; i < n; i++) out.push(new ArithmeticModel(symbols)); return out; }
|
|
64
|
+
// A model made on first use: a fresh model is the same whenever it is made, and most
|
|
65
|
+
// of the 256 per table are never touched in a chunk
|
|
66
|
+
function lazyModel(table, i) { return table[i] || (table[i] = new ArithmeticModel(256)); }
|
|
64
67
|
function medians() { var out = []; for (var i = 0; i < 16; i++) out.push(new StreamingMedian()); return out; }
|
|
65
68
|
|
|
66
69
|
/** Point10 v2 decompressor: one per chunk. */
|
|
@@ -72,9 +75,9 @@ export function Point10Decompressor() {
|
|
|
72
75
|
this.lastHeight = new Int32Array(8);
|
|
73
76
|
this.changedValues = new ArithmeticModel(64);
|
|
74
77
|
this.scanAngleRank = models(2, 256);
|
|
75
|
-
this.bitByte =
|
|
76
|
-
this.classification =
|
|
77
|
-
this.userData =
|
|
78
|
+
this.bitByte = new Array(256);
|
|
79
|
+
this.classification = new Array(256);
|
|
80
|
+
this.userData = new Array(256);
|
|
78
81
|
this.icIntensity = new IntegerDecompressor(16, 4);
|
|
79
82
|
this.icPointSourceId = new IntegerDecompressor(16, 1);
|
|
80
83
|
this.icDx = new IntegerDecompressor(32, 2);
|
|
@@ -98,7 +101,7 @@ Point10Decompressor.prototype.next = function (dec, view, o) {
|
|
|
98
101
|
var changed = dec.decodeSymbol(this.changedValues);
|
|
99
102
|
var r, n, m, l;
|
|
100
103
|
if (changed !== 0) {
|
|
101
|
-
if (changed & 32) this.bits = dec.decodeSymbol(this.bitByte
|
|
104
|
+
if (changed & 32) this.bits = dec.decodeSymbol(lazyModel(this.bitByte, this.bits));
|
|
102
105
|
r = this.bits & 7; n = (this.bits >>> 3) & 7;
|
|
103
106
|
m = NUMBER_RETURN_MAP[n][r]; l = NUMBER_RETURN_LEVEL[n][r];
|
|
104
107
|
if (changed & 16) {
|
|
@@ -107,12 +110,12 @@ Point10Decompressor.prototype.next = function (dec, view, o) {
|
|
|
107
110
|
} else {
|
|
108
111
|
this.intensity = this.lastIntensity[m];
|
|
109
112
|
}
|
|
110
|
-
if (changed & 8) this.cls = dec.decodeSymbol(this.classification
|
|
113
|
+
if (changed & 8) this.cls = dec.decodeSymbol(lazyModel(this.classification, this.cls));
|
|
111
114
|
if (changed & 4) {
|
|
112
115
|
var d = dec.decodeSymbol(this.scanAngleRank[(this.bits >>> 6) & 1]);
|
|
113
116
|
this.angle = ((this.angle + d) << 24) >> 24; // i8 wrapping_add
|
|
114
117
|
}
|
|
115
|
-
if (changed & 2) this.user = dec.decodeSymbol(this.userData
|
|
118
|
+
if (changed & 2) this.user = dec.decodeSymbol(lazyModel(this.userData, this.user));
|
|
116
119
|
if (changed & 1) this.psid = this.icPointSourceId.decompress(dec, this.psid, 0);
|
|
117
120
|
} else {
|
|
118
121
|
r = this.bits & 7; n = (this.bits >>> 3) & 7;
|
package/src/laz/v3/layered.js
CHANGED
|
@@ -23,12 +23,13 @@ var LE = true;
|
|
|
23
23
|
|
|
24
24
|
// laz-rs's context switch, shared by the three items: returns the index whose
|
|
25
25
|
// last value predicts this point (`last_idx`), after switching to `ctx`.
|
|
26
|
-
|
|
26
|
+
// item.seed(from, to) starts a context from another's last value.
|
|
27
|
+
function enterContext(item, ctx) {
|
|
27
28
|
var lastIdx = item.lastContext;
|
|
28
29
|
if (item.lastContext !== ctx) {
|
|
29
30
|
item.lastContext = ctx;
|
|
30
31
|
if (item.unused[ctx]) {
|
|
31
|
-
seed(lastIdx, ctx);
|
|
32
|
+
item.seed(lastIdx, ctx);
|
|
32
33
|
item.unused[ctx] = false;
|
|
33
34
|
if (item.version === 3) lastIdx = ctx;
|
|
34
35
|
}
|
|
@@ -65,9 +66,9 @@ Rgb14Decompressor.prototype.first = function (src, s, view, o, ctx) {
|
|
|
65
66
|
};
|
|
66
67
|
Rgb14Decompressor.prototype.readLayerSizes = function (dv, q) { this.layerSize = dv.getUint32(q, LE); return q + 4; };
|
|
67
68
|
Rgb14Decompressor.prototype.readLayers = function (bytes, q, e, fail, base) { this.dec = layer(this, bytes, q, this.layerSize, fail, e, base); return q + this.layerSize; };
|
|
69
|
+
Rgb14Decompressor.prototype.seed = function (from, to) { this.last[to].set(this.last[from]); };
|
|
68
70
|
Rgb14Decompressor.prototype.next = function (view, o, ctx) {
|
|
69
|
-
var
|
|
70
|
-
var idx = enterContext(this, ctx.v, function (from, to) { self.last[to].set(self.last[from]); });
|
|
71
|
+
var idx = enterContext(this, ctx.v);
|
|
71
72
|
var rgb = this.last[idx];
|
|
72
73
|
if (this.dec) decodeRgb(this.dec, this.models[this.lastContext], rgb);
|
|
73
74
|
view.setUint16(o, rgb[0], LE); view.setUint16(o + 2, rgb[1], LE); view.setUint16(o + 4, rgb[2], LE);
|
|
@@ -93,9 +94,9 @@ Nir14Decompressor.prototype.first = function (src, s, view, o, ctx) {
|
|
|
93
94
|
};
|
|
94
95
|
Nir14Decompressor.prototype.readLayerSizes = Rgb14Decompressor.prototype.readLayerSizes;
|
|
95
96
|
Nir14Decompressor.prototype.readLayers = Rgb14Decompressor.prototype.readLayers;
|
|
97
|
+
Nir14Decompressor.prototype.seed = function (from, to) { this.last[to] = this.last[from]; };
|
|
96
98
|
Nir14Decompressor.prototype.next = function (view, o, ctx) {
|
|
97
|
-
var
|
|
98
|
-
var idx = enterContext(this, ctx.v, function (from, to) { self.last[to] = self.last[from]; });
|
|
99
|
+
var idx = enterContext(this, ctx.v);
|
|
99
100
|
if (this.dec) {
|
|
100
101
|
var c = this.lastContext, last = this.last[idx];
|
|
101
102
|
var sym = this.dec.decodeSymbol(this.used[c]);
|
|
@@ -134,9 +135,9 @@ Byte14Decompressor.prototype.readLayers = function (bytes, q, e, fail, base) {
|
|
|
134
135
|
for (var i = 0; i < this.size; i++) { this.dec[i] = layer(this, bytes, q, this.sizes[i], fail, e, base); q += this.sizes[i]; }
|
|
135
136
|
return q;
|
|
136
137
|
};
|
|
138
|
+
Byte14Decompressor.prototype.seed = function (from, to) { this.models[to] = byteModels(this.size); this.last[to].set(this.last[from]); };
|
|
137
139
|
Byte14Decompressor.prototype.next = function (view, o, ctx) {
|
|
138
|
-
var
|
|
139
|
-
var idx = enterContext(this, ctx.v, function (from, to) { self.models[to] = byteModels(self.size); self.last[to].set(self.last[from]); });
|
|
140
|
+
var idx = enterContext(this, ctx.v);
|
|
140
141
|
var last = this.last[idx], models = this.models[ctx.v];
|
|
141
142
|
for (var i = 0; i < this.size; i++) {
|
|
142
143
|
if (this.dec[i]) last[i] = (last[i] + this.dec[i].decodeSymbol(models[i])) & 255;
|