gds-lens 0.1.0

package/src/layout-bytes.js

@@ -0,0 +1,143 @@
+// Turns the bytes read off disk into the bytes the viewer parses: a gzipped
+// layout (`.gds.gz` and friends) is expanded here, and everything downstream --
+// the webview, the parse Worker, the wasm module's own GDSII/OASIS sniffing --
+// sees exactly what an uncompressed file would have produced.
+//
+// Expanding out here rather than inside the wasm module is a memory decision.
+// The viewer's entire size budget is the 32-bit wasm heap: the file's bytes and
+// the flattened geometry built from them both have to fit in 4 GB (see "Limits"
+// in README.md), and that heap is the one address space that can least afford a
+// second full copy of the file. JavaScript has no such ceiling, so doing it
+// here spends the compressed copy and the decompressor's scratch space where
+// they cost nothing, and hands the wasm side a single buffer.
+//
+// Built on the platform's DecompressionStream rather than Node's zlib, so the
+// same code runs in a browser, a Worker and Node unchanged. That makes
+// expansion async, which is the one shape change from a zlib version; it also
+// makes the size cap ours to enforce, since there is no `maxOutputLength` to
+// hand off to -- that's what the running total in gunzip() is for.
+//
+// No imports, no DOM and no wasm, the same shape as marker-parsers.js and
+// load-errors.js. Published on its own as `gds-lens/layout-bytes`.
+"use strict";
+
+// gzip's magic number: RFC 1952's ID1/ID2. Detection is by content rather than
+// by a ".gz" on the name, matching how GDSII vs OASIS is already decided (see
+// detect_format in gds_common.hpp). That way a layout named with an unexpected
+// extension still loads, and so does a plain ".gds" that is secretly gzipped --
+// which is how these arrive out of some flows.
+const GZIP_ID1 = 0x1f;
+const GZIP_ID2 = 0x8b;
+
+// Smallest possible gzip member: a 10-byte header plus an 8-byte trailer.
+const GZIP_MIN_BYTES = 18;
+
+function looksGzipped(bytes) {
+    return !!bytes && bytes.length >= 2 && bytes[0] === GZIP_ID1 && bytes[1] === GZIP_ID2;
+}
+
+// The uncompressed size gzip records in its trailer (RFC 1952's ISIZE), or null
+// if the input is too short to hold one.
+//
+// Only ever used to word a message. It's stored modulo 2^32, and for a
+// multi-member file it describes the last member alone, so it is a hint about
+// the file rather than a fact about it -- which is exactly why it isn't what
+// enforces the limit below. The running total in gunzip() does that, by stopping
+// the moment the output overruns instead of trusting what the file claims about
+// itself.
+//
+// Read by hand rather than with Buffer.readUInt32LE: the caller's bytes come
+// from vscode.workspace.fs.readFile, which yields a plain Uint8Array. The top
+// byte is multiplied rather than shifted, since `<< 24` would make sizes over
+// 2 GB come back negative.
+function gzipStoredSize(bytes) {
+    if (!bytes || bytes.length < GZIP_MIN_BYTES) return null;
+    const end = bytes.length;
+    return bytes[end - 4] + bytes[end - 3] * 0x100 + bytes[end - 2] * 0x10000 +
+           bytes[end - 1] * 0x1000000;
+}
+
+// Expands one gzip buffer, refusing to accumulate more than `cap` bytes.
+//
+// The write side is started but deliberately not awaited before reading. A
+// DecompressionStream applies backpressure, so `writer.write()` of a whole
+// layout doesn't settle until the read loop below has drained it -- awaiting it
+// first would deadlock. Its rejection is swallowed because every failure the
+// stream can have (a bad header, a truncated body, corrupt deflate data) also
+// surfaces from `reader.read()`, which is where the throw wants to come from;
+// without the catch, the same error would additionally go unhandled here.
+async function gunzip(bytes, cap) {
+    const stream = new DecompressionStream("gzip");
+    const writer = stream.writable.getWriter();
+    const pump = writer.write(bytes).then(() => writer.close()).catch(() => {});
+
+    const reader = stream.readable.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        total += value.byteLength;
+        // `>` rather than `>=`: a file expanding to exactly the cap still fits.
+        if (total > cap) {
+            await reader.cancel();
+            await pump;
+            const err = new Error(`expands past the ${cap} byte limit`);
+            err.tooLarge = true;
+            throw err;
+        }
+        chunks.push(value);
+    }
+    await pump;
+
+    // A layout that arrived in one chunk is handed straight back rather than
+    // copied into a second buffer of the same size. At these sizes the copy is
+    // the expensive part, not the bookkeeping.
+    if (chunks.length === 1) return chunks[0];
+    const out = new Uint8Array(total);
+    let at = 0;
+    for (const chunk of chunks) {
+        out.set(chunk, at);
+        at += chunk.byteLength;
+    }
+    return out;
+}
+
+// Uncompressed layout bytes, whatever the input was.
+//
+//   { ok: true,  bytes, gzipped, storedSize }
+//   { ok: false, reason: "too-large" | "corrupt", storedSize, limit, detail }
+//
+// Not-gzipped input is passed straight back, untouched and uncopied, so the
+// ordinary path costs one two-byte comparison. maxBytes caps what a compressed
+// file is allowed to expand to; pass a non-finite value for no cap. The reason
+// codes are split because the two failures need different things said about
+// them -- one is about this machine's limits, the other about the file being
+// broken or half-written -- and the prose for both lives with the caller's other
+// messages rather than here.
+async function decodeLayoutBytes(bytes, maxBytes) {
+    if (!looksGzipped(bytes)) return { ok: true, bytes: bytes, gzipped: false, storedSize: null };
+
+    const storedSize = gzipStoredSize(bytes);
+    const cap = Number.isFinite(maxBytes) ? maxBytes : Infinity;
+    try {
+        return { ok: true, bytes: await gunzip(bytes, cap), gzipped: true, storedSize: storedSize };
+    } catch (err) {
+        // The `tooLarge` flag is the cap above being hit -- the one failure here
+        // that's about size rather than about the data. Everything else is the
+        // decompressor refusing the stream itself, which means the compressed
+        // data can't be read at all.
+        return {
+            ok: false,
+            reason: err && err.tooLarge ? "too-large" : "corrupt",
+            storedSize: storedSize,
+            // Reported back rather than left for the caller to remember: it is
+            // the one number a "too large" message needs, and the caller that
+            // passed it is not always the one wording the failure.
+            limit: cap,
+            detail: err && err.message ? err.message : String(err)
+        };
+    }
+}
+
+export { looksGzipped, gzipStoredSize, decodeLayoutBytes };

package/src/load-errors.js

@@ -0,0 +1,64 @@
+// Turns the engine-level failures a layout load can hit into text a layout
+// engineer can act on.
+//
+// Imported by both sides of the load: the main thread (viewer.js) and the
+// parse Worker, which are separate script contexts and each get their own
+// copy through the bundler. No imports of its own, and no DOM or wasm, so
+// Node unit tests import it directly too.
+//
+// Running out of room in the wasm heap surfaces as one of a handful of
+// unhelpful strings depending on which allocation happened to be the one that
+// failed -- "memory access out of bounds" when a bounds check catches it
+// first, "Aborted()" when Emscripten's allocator gives up, a RangeError when
+// it's a JS-side typed array. They all mean the same thing to the user.
+const OOM_PATTERN = /memory access out of bounds|Cannot enlarge memory|Aborted\(|out of memory|Array buffer allocation failed|Invalid (typed )?array length/i;
+
+const OOM_MESSAGE =
+    "Out of memory: this layout is too large to open.\n\n" +
+    "The viewer parses layouts in a 32-bit WebAssembly module, so the whole " +
+    "flattened design has to fit in 4 GB. Layouts that reuse cells (arrays " +
+    "and repeated placements) go much further than fully flattened ones, " +
+    "because a repeated cell is drawn as GPU instances instead of being " +
+    "copied for every placement.";
+
+// `prefix` labels non-memory failures with where they came from (the Worker
+// passes one); out-of-memory keeps its own wording either way, since the
+// cause is the layout rather than the component that happened to notice.
+function describeLoadFailure(err, prefix) {
+    const text = err && err.message ? err.message : String(err);
+    if (OOM_PATTERN.test(text)) return `${OOM_MESSAGE}\n\n(${text})`;
+    return prefix ? `${prefix}: ${text}` : text;
+}
+
+function isOutOfMemory(err) {
+    return OOM_PATTERN.test(err && err.message ? err.message : String(err));
+}
+
+// Wording for a failed gzip expansion (the {ok:false} half of what
+// decodeLayoutBytes returns; see layout-bytes.js, which deliberately leaves
+// the prose to its caller).
+//
+// The two reasons need different things said. "too-large" is about this
+// machine: the file is fine, there is nowhere to put it, and the number worth
+// quoting is the limit. "corrupt" is about the file: it is truncated or was
+// written by something that stopped halfway, and no limit would have helped.
+function describeDecodeFailure(result) {
+    if (!result || result.ok) return "";
+    if (result.reason === "too-large") {
+        const gb = (result.limit / (1024 * 1024 * 1024)).toFixed(1);
+        const claimed = result.storedSize
+            ? ` The file's own trailer claims ${(result.storedSize / (1024 * 1024)).toFixed(0)} MB uncompressed.`
+            : "";
+        return "This compressed layout is too large to expand.\n\n"
+            + `Expanding stopped at the ${gb} GB limit.${claimed} The viewer parses `
+            + "layouts in a 32-bit WebAssembly module, so the expanded file and the "
+            + "geometry built from it both have to fit in one 4 GB address space. "
+            + "An uncompressed copy of the same design will not help; a design that "
+            + "reuses cells rather than flattening them will.";
+    }
+    return "This layout looks gzipped, but the compressed data could not be read.\n\n"
+        + "Usually a truncated or half-written file -- a copy that was interrupted, "
+        + `or a download that stopped early. (${result.detail})`;
+}
+
+export { describeLoadFailure, isOutOfMemory, describeDecodeFailure };