memory-extract 0.1.0

package/tools/glob.mjs

@@ -0,0 +1,12 @@
+export const globToRegExp = (pattern) => {
+    const escaped = pattern
+        .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*\*/g, "::DOUBLESTAR::")
+        .replace(/\*/g, "[^/]*")
+        .replace(/::DOUBLESTAR::/g, ".*");
+
+    return new RegExp(`^${escaped}$`);
+};
+
+export const matchesAnyGlob = (relativePath, patterns) =>
+    patterns.some((pattern) => globToRegExp(pattern).test(relativePath));

package/tools/hostProject.mjs

@@ -0,0 +1,45 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+const DEFAULT_EXCLUDE = ["**/*.map"];
+
+export const loadHostProject = async (rootDir = process.cwd()) => {
+    try {
+        const packagePath = path.join(rootDir, "package.json");
+        const pkg = JSON.parse(await readFile(packagePath, "utf8"));
+        const memory = pkg.memory ?? {};
+        const name = memory.name ?? pkg.name ?? "memory";
+        const dist = path.resolve(rootDir, memory.dist ?? "dist");
+        const out = memory.out
+            ? path.resolve(rootDir, memory.out)
+            : path.join(dist, `${name}.png`);
+
+        return {
+            name,
+            dist,
+            out,
+            cover: memory.cover ? path.resolve(rootDir, memory.cover) : "",
+            entry: memory.entry ?? "index.html",
+            exclude: [...new Set([...DEFAULT_EXCLUDE, ...(memory.exclude ?? [])])],
+            coverMaxSize: memory.coverMaxSize ?? 512,
+            coverResize: memory.coverResize !== false,
+            cache: memory.cache !== false,
+            unpackDir: memory.unpackDir
+                ? path.resolve(rootDir, memory.unpackDir)
+                : path.join(dist, "unpack"),
+        };
+    } catch {
+        return {
+            name: "memory",
+            dist: path.resolve(rootDir, "dist"),
+            out: path.resolve(rootDir, "dist", "memory.png"),
+            cover: "",
+            entry: "index.html",
+            exclude: [...DEFAULT_EXCLUDE],
+            coverMaxSize: 512,
+            coverResize: true,
+            cache: true,
+            unpackDir: path.resolve(rootDir, "dist", "unpack"),
+        };
+    }
+};

package/tools/memoryFormat.mjs

@@ -0,0 +1,277 @@
+import { gunzipSync } from "node:zlib";
+import path from "node:path";
+import {
+    assertSupportedVersion,
+    buildV2Manifest,
+    decodeV2Archive,
+    encodeV2Archive,
+    guessMimeType,
+    listManifestFiles,
+    MAX_PAYLOAD_SIZE,
+    MEMORY_VERSION,
+    readManifestFile,
+    SUPPORTED_VERSIONS,
+} from "../src/payloadFormat.js";
+import { PNG_SIGNATURE, createPngChunk, gzipPayload } from "./pngUtils.mjs";
+
+export {
+    assertSupportedVersion,
+    guessMimeType,
+    listManifestFiles,
+    MAX_PAYLOAD_SIZE,
+    MEMORY_VERSION,
+    SUPPORTED_VERSIONS,
+} from "../src/payloadFormat.js";
+
+export const MEMORY_MAGIC = Buffer.from("WLFC");
+export const MEMORY_CHUNK_TYPE = Buffer.from("wLFC");
+
+const parsePngChunks = (buffer) => {
+    const view = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+
+    if (view.length < 8 || !view.subarray(0, 8).equals(PNG_SIGNATURE)) {
+        throw new Error("Invalid PNG signature.");
+    }
+
+    const chunks = [];
+    let offset = 8;
+
+    while (offset + 12 <= view.length) {
+        const length = view.readUInt32BE(offset);
+        const type = view.subarray(offset + 4, offset + 8);
+        const dataStart = offset + 8;
+        const dataEnd = dataStart + length;
+
+        if (dataEnd + 4 > view.length) {
+            throw new Error("PNG chunk exceeds file size.");
+        }
+
+        chunks.push({
+            type: type.toString("ascii"),
+            data: view.subarray(dataStart, dataEnd),
+            start: offset,
+            end: dataEnd + 4,
+        });
+
+        offset = dataEnd + 4;
+
+        if (type.equals(Buffer.from("IEND"))) {
+            break;
+        }
+    }
+
+    return { view, chunks, trailing: view.subarray(offset) };
+};
+
+const readMemoryPayload = (chunkData) => {
+    if (chunkData.length < 8) {
+        throw new Error("Memory chunk is too small.");
+    }
+
+    if (!chunkData.subarray(0, 4).equals(MEMORY_MAGIC)) {
+        throw new Error("Memory chunk magic mismatch.");
+    }
+
+    const version = chunkData.readUInt32BE(4);
+    assertSupportedVersion(version);
+
+    const payload = chunkData.subarray(8);
+    if (payload.length > MAX_PAYLOAD_SIZE) {
+        throw new Error(`Memory payload exceeds ${MAX_PAYLOAD_SIZE} bytes.`);
+    }
+
+    return { version, payload };
+};
+
+const extractLegacyFooter = (view) => {
+    const magicIndex = view.lastIndexOf(MEMORY_MAGIC);
+    if (magicIndex < 8) {
+        throw new Error("Memory footer not found.");
+    }
+
+    const version = view.readUInt32BE(magicIndex + 4);
+    const payloadLength = view.readUInt32BE(magicIndex + 8);
+    const payloadStart = magicIndex + 12;
+    const payloadEnd = payloadStart + payloadLength;
+
+    if (version !== 1) {
+        throw new Error(`Unsupported legacy memory version: ${version}`);
+    }
+
+    if (payloadEnd > view.length) {
+        throw new Error("Memory payload length exceeds file size.");
+    }
+
+    return {
+        png: view.subarray(0, magicIndex),
+        payload: view.subarray(payloadStart, payloadEnd),
+        version,
+    };
+};
+
+export const buildManifest = ({
+    name,
+    entry,
+    files,
+    kind = "web-app",
+    runtime = "iframe-sandbox",
+}) => ({
+    v: 1,
+    kind,
+    name,
+    entry,
+    runtime,
+    files,
+});
+
+export const encodeManifest = (manifest) =>
+    gzipPayload(Buffer.from(JSON.stringify(manifest), "utf8"));
+
+export const decodeManifest = (payload) =>
+    JSON.parse(gunzipSync(payload).toString("utf8"));
+
+export const encodeMemoryPayload = ({
+    name,
+    entry,
+    files,
+    kind = "web-app",
+    runtime = "iframe-sandbox",
+    version = MEMORY_VERSION,
+}) => {
+    if (version === 1) {
+        const encodedFiles = {};
+
+        for (const [filePath, buffer] of Object.entries(files)) {
+            encodedFiles[filePath] = encodeManifestFile(buffer, filePath);
+        }
+
+        const manifest = buildManifest({ name, entry, files: encodedFiles, kind, runtime });
+
+        return {
+            version: 1,
+            manifest,
+            payload: encodeManifest(manifest),
+        };
+    }
+
+    if (version !== 2) {
+        throw new Error(`Unsupported memory version: ${version}`);
+    }
+
+    const archive = encodeV2Archive({ name, entry, files, kind, runtime });
+    const { manifest } = buildV2Manifest({ name, entry, files, kind, runtime });
+
+    return {
+        version: 2,
+        manifest,
+        payload: gzipPayload(Buffer.from(archive)),
+    };
+};
+
+export const decodeMemoryPayload = (payload, version) => {
+    assertSupportedVersion(version);
+
+    if (version === 1) {
+        return {
+            manifest: decodeManifest(payload),
+            fileBytes: null,
+        };
+    }
+
+    return decodeV2Archive(gunzipSync(payload));
+};
+
+export const decodeManifestFile = (fileRecord) => {
+    if (!fileRecord?.data) {
+        throw new Error("Invalid manifest file record.");
+    }
+
+    return Buffer.from(fileRecord.data, fileRecord.encoding ?? "base64");
+};
+
+export { readManifestFile };
+
+export const readMemoryFile = (manifest, filePath, fileBytes = null) =>
+    readManifestFile(manifest, filePath, fileBytes, decodeManifestFile);
+
+export const encodeManifestFile = (buffer, filePath = "") => ({
+    encoding: "base64",
+    mime: guessMimeType(filePath),
+    data: Buffer.from(buffer).toString("base64"),
+});
+
+export const embedMemory = (pngBuffer, payloadBuffer, version = MEMORY_VERSION) => {
+    if (!Buffer.isBuffer(pngBuffer) || pngBuffer.length === 0) {
+        throw new Error("Cover PNG buffer is empty.");
+    }
+
+    if (!Buffer.isBuffer(payloadBuffer) || payloadBuffer.length === 0) {
+        throw new Error("Payload buffer is empty.");
+    }
+
+    assertSupportedVersion(version);
+
+    const { view, chunks } = parsePngChunks(pngBuffer);
+    const iendChunk = chunks.find((chunk) => chunk.type === "IEND");
+
+    if (!iendChunk) {
+        throw new Error("PNG IEND chunk not found.");
+    }
+
+    const chunkData = Buffer.alloc(8 + payloadBuffer.length);
+    MEMORY_MAGIC.copy(chunkData, 0);
+    chunkData.writeUInt32BE(version, 4);
+    payloadBuffer.copy(chunkData, 8);
+
+    const memoryChunk = createPngChunk(MEMORY_CHUNK_TYPE, chunkData);
+
+    return Buffer.concat([
+        view.subarray(0, iendChunk.start),
+        memoryChunk,
+        view.subarray(iendChunk.start),
+    ]);
+};
+
+const decodeExtractedMemory = (png, payload, version) => {
+    const { manifest, fileBytes } = decodeMemoryPayload(payload, version);
+
+    return {
+        png,
+        payload,
+        manifest,
+        fileBytes,
+        version,
+    };
+};
+
+export const extractMemory = (buffer) => {
+    const view = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+
+    if (view.length < 12) {
+        throw new Error("File is too small to be a memory PNG.");
+    }
+
+    try {
+        const { view: pngView, chunks, trailing } = parsePngChunks(view);
+        const memoryChunk = chunks.find((chunk) => chunk.type === "wLFC");
+
+        if (memoryChunk) {
+            const { version, payload } = readMemoryPayload(memoryChunk.data);
+            return decodeExtractedMemory(pngView, payload, version);
+        }
+
+        if (trailing.length > 0) {
+            const legacy = extractLegacyFooter(view);
+            return decodeExtractedMemory(legacy.png, legacy.payload, legacy.version);
+        }
+    } catch (error) {
+        if (error.message?.includes("Memory footer not found")) {
+            throw error;
+        }
+
+        const legacy = extractLegacyFooter(view);
+        return decodeExtractedMemory(legacy.png, legacy.payload, legacy.version);
+    }
+
+    throw new Error("Memory chunk not found.");
+};

package/tools/pack.mjs

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+import { readdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { createBlankCover } from "./blankCover.mjs";
+import { matchesAnyGlob } from "./glob.mjs";
+import { loadHostProject } from "./hostProject.mjs";
+import {
+    buildPackFingerprint,
+    isPackUpToDate,
+    savePackCache,
+} from "./packCache.mjs";
+import { prepareCover } from "./resizeCover.mjs";
+import {
+    embedMemory,
+    encodeMemoryPayload,
+} from "./memoryFormat.mjs";
+
+const shouldSkipPath = (absolutePath, skipPaths) => {
+    const resolved = path.resolve(absolutePath);
+
+    return skipPaths.some((skipPath) => {
+        const resolvedSkip = path.resolve(skipPath);
+        return (
+            resolved === resolvedSkip ||
+            resolved.startsWith(`${resolvedSkip}${path.sep}`)
+        );
+    });
+};
+
+const collectFiles = async (dir, baseDir, skipPaths, excludePatterns) => {
+    const entries = await readdir(dir, { withFileTypes: true });
+    const files = {};
+    const fileReads = [];
+    const subdirs = [];
+
+    for (const entry of entries) {
+        const absolutePath = path.join(dir, entry.name);
+
+        if (shouldSkipPath(absolutePath, skipPaths)) {
+            continue;
+        }
+
+        if (entry.isDirectory()) {
+            subdirs.push(absolutePath);
+            continue;
+        }
+
+        const relativePath = path.relative(baseDir, absolutePath)
+            .split(path.sep)
+            .join("/");
+
+        if (matchesAnyGlob(relativePath, excludePatterns)) {
+            continue;
+        }
+
+        fileReads.push(
+            readFile(absolutePath).then((buffer) => {
+                files[relativePath] = buffer;
+            })
+        );
+    }
+
+    await Promise.all([
+        ...fileReads,
+        ...subdirs.map((subdir) =>
+            collectFiles(subdir, baseDir, skipPaths, excludePatterns).then(
+                (nested) => {
+                    Object.assign(files, nested);
+                }
+            )
+        ),
+    ]);
+
+    return files;
+};
+
+const buildSkipPaths = (options) =>
+    [options.out, path.join(path.resolve(options.dist), "unpack")].map((entry) =>
+        path.resolve(entry)
+    );
+
+const parseArgs = async (argv) => {
+    const rootDir = process.cwd();
+    const host = await loadHostProject(rootDir);
+    const options = {
+        root: rootDir,
+        dist: host.dist,
+        cover: host.cover,
+        out: host.out,
+        name: host.name,
+        entry: host.entry,
+        exclude: [...host.exclude],
+        coverMaxSize: host.coverMaxSize,
+        coverResize: host.coverResize,
+        cache: host.cache,
+        force: false,
+        check: false,
+    };
+
+    for (let index = 0; index < argv.length; index += 1) {
+        const arg = argv[index];
+        if (arg === "--root") {
+            options.root = path.resolve(argv[index + 1] ?? "");
+            index += 1;
+        } else if (arg === "--dist") {
+            options.dist = path.resolve(argv[index + 1] ?? "");
+            index += 1;
+        } else if (arg === "--cover") {
+            options.cover = path.resolve(argv[index + 1] ?? "");
+            index += 1;
+        } else if (arg === "--out") {
+            options.out = path.resolve(argv[index + 1] ?? "");
+            index += 1;
+        } else if (arg === "--name") {
+            options.name = argv[index + 1] ?? options.name;
+            index += 1;
+        } else if (arg === "--entry") {
+            options.entry = argv[index + 1] ?? options.entry;
+            index += 1;
+        } else if (arg === "--exclude") {
+            options.exclude.push(argv[index + 1] ?? "");
+            index += 1;
+        } else if (arg === "--cover-max-size") {
+            options.coverMaxSize = Number(argv[index + 1] ?? options.coverMaxSize);
+            index += 1;
+        } else if (arg === "--no-cover-resize") {
+            options.coverResize = false;
+        } else if (arg === "--no-cache") {
+            options.cache = false;
+        } else if (arg === "--force") {
+            options.force = true;
+        } else if (arg === "--check") {
+            options.check = true;
+        }
+    }
+
+    options.exclude = [...new Set(options.exclude.filter(Boolean))];
+
+    return options;
+};
+
+const formatKb = (bytes) => `${(bytes / 1024).toFixed(1)} KB`;
+
+const main = async () => {
+    const options = await parseArgs(process.argv.slice(2));
+
+    try {
+        await stat(options.dist);
+    } catch {
+        throw new Error(`Build output not found at ${options.dist}. Run your build first.`);
+    }
+
+    let cover;
+    if (options.cover) {
+        const sourceCover = await readFile(options.cover);
+        const prepared = await prepareCover(sourceCover, {
+            maxSize: options.coverMaxSize,
+            resize: options.coverResize,
+        });
+
+        cover = prepared.buffer;
+
+        if (prepared.changed && prepared.from && prepared.info) {
+            console.log(
+                `Resized cover from ${prepared.from.width}x${prepared.from.height} (${formatKb(prepared.from.bytes)}) to ${prepared.info.width}x${prepared.info.height} (${formatKb(prepared.info.bytes)})`
+            );
+        }
+
+        console.log(`Using cover image ${options.cover}`);
+    } else {
+        cover = createBlankCover();
+        console.log("Using blank cover image");
+    }
+
+    const skipPaths = buildSkipPaths(options);
+    const files = await collectFiles(
+        options.dist,
+        options.dist,
+        skipPaths,
+        options.exclude
+    );
+
+    if (!files[options.entry]) {
+        throw new Error(
+            `Entry not found in dist: ${options.entry}. Checked ${Object.keys(files).length} files.`
+        );
+    }
+
+    const fingerprint = buildPackFingerprint(options, files, cover);
+
+    if (options.check && !options.cache) {
+        throw new Error("Pack cache is disabled. Remove --check or enable cache in memory config.");
+    }
+
+    const upToDate =
+        options.cache &&
+        !options.force &&
+        (await isPackUpToDate(options.root, fingerprint, options.out));
+
+    if (options.check) {
+        if (upToDate) {
+            console.log(`Memory pack is up to date: ${options.out}`);
+            return;
+        }
+
+        console.error(`Memory pack is stale: ${options.out}`);
+        process.exit(1);
+    }
+
+    if (upToDate) {
+        console.log(`Memory unchanged, skipping pack: ${options.out}`);
+        return;
+    }
+
+    const { payload, version } = encodeMemoryPayload({
+        name: options.name,
+        entry: options.entry,
+        files,
+    });
+    const memory = embedMemory(cover, payload, version);
+
+    const tempPath = `${options.out}.tmp`;
+    await writeFile(tempPath, memory);
+    await rename(tempPath, options.out);
+
+    if (options.cache) {
+        await savePackCache(options.root, fingerprint);
+    }
+
+    console.log(
+        `Packed ${Object.keys(files).length} files into ${options.out} (${formatKb(memory.length)})`
+    );
+    console.log(`  cover: ${formatKb(cover.length)}, payload: ${formatKb(payload.length)}, format: v${version}`);
+};
+
+main().catch((error) => {
+    console.error(error.message ?? error);
+    process.exit(1);
+});

package/tools/packCache.mjs

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+import { readFile, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { MEMORY_VERSION } from "./memoryFormat.mjs";
+
+export const CACHE_FILE = ".memory-pack-cache.json";
+
+export const hashBuffer = (buffer) =>
+    createHash("sha256").update(buffer).digest("hex");
+
+export const buildPackFingerprint = (options, files, cover) => ({
+    cacheVersion: 1,
+    formatVersion: MEMORY_VERSION,
+    name: options.name,
+    entry: options.entry,
+    out: path.resolve(options.out),
+    exclude: [...options.exclude].sort(),
+    coverMaxSize: options.coverMaxSize,
+    coverResize: options.coverResize,
+    cover: hashBuffer(cover),
+    files: Object.fromEntries(
+        Object.keys(files)
+            .sort()
+            .map((filePath) => [filePath, hashBuffer(files[filePath])])
+    ),
+});
+
+export const fingerprintsEqual = (left, right) =>
+    JSON.stringify(left) === JSON.stringify(right);
+
+export const cachePathFor = (rootDir) => path.join(rootDir, CACHE_FILE);
+
+export const loadPackCache = async (rootDir) => {
+    try {
+        const raw = await readFile(cachePathFor(rootDir), "utf8");
+        return JSON.parse(raw);
+    } catch {
+        return null;
+    }
+};
+
+export const savePackCache = async (rootDir, fingerprint) => {
+    await writeFile(
+        cachePathFor(rootDir),
+        `${JSON.stringify({ fingerprint }, null, 2)}\n`
+    );
+};
+
+export const isPackUpToDate = async (rootDir, fingerprint, outputPath) => {
+    try {
+        await stat(outputPath);
+    } catch {
+        return false;
+    }
+
+    const cache = await loadPackCache(rootDir);
+    if (!cache?.fingerprint) {
+        return false;
+    }
+
+    return fingerprintsEqual(cache.fingerprint, fingerprint);
+};

package/tools/pngInspect.mjs

@@ -0,0 +1,22 @@
+const PNG_SIGNATURE = Buffer.from([
+    0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+]);
+
+export const inspectPng = (buffer) => {
+    const view = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+
+    if (view.length < 24 || !view.subarray(0, 8).equals(PNG_SIGNATURE)) {
+        return null;
+    }
+
+    const chunkType = view.subarray(12, 16).toString("ascii");
+    if (chunkType !== "IHDR") {
+        return null;
+    }
+
+    return {
+        width: view.readUInt32BE(16),
+        height: view.readUInt32BE(20),
+        bytes: view.length,
+    };
+};

package/tools/pngUtils.mjs

@@ -0,0 +1,36 @@
+import { gzipSync } from "node:zlib";
+
+const crcTable = (() => {
+    const table = new Uint32Array(256);
+    for (let index = 0; index < 256; index += 1) {
+        let value = index;
+        for (let bit = 0; bit < 8; bit += 1) {
+            value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
+        }
+        table[index] = value >>> 0;
+    }
+    return table;
+})();
+
+export const crc32 = (buffer) => {
+    let value = 0xffffffff;
+    for (let index = 0; index < buffer.length; index += 1) {
+        value = crcTable[(value ^ buffer[index]) & 0xff] ^ (value >>> 8);
+    }
+    return (value ^ 0xffffffff) >>> 0;
+};
+
+export const createPngChunk = (type, data) => {
+    const length = Buffer.alloc(4);
+    length.writeUInt32BE(data.length);
+    const typeBuffer = Buffer.isBuffer(type) ? type : Buffer.from(type);
+    const crc = Buffer.alloc(4);
+    crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])));
+    return Buffer.concat([length, typeBuffer, data, crc]);
+};
+
+export const PNG_SIGNATURE = Buffer.from([
+    0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+]);
+
+export const gzipPayload = (buffer) => gzipSync(buffer, { level: 9 });