@johannc-dev/zyrk 0.0.0-stage → 0.2.0

package/dist/local-storage.js

@@ -0,0 +1,499 @@
+import { createHash, randomBytes } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
+import { join } from "node:path";
+const MIB = 1024 * 1024;
+export const localStorageLimits = {
+    maxObjectBytes: 64 * MIB,
+    chunkBytes: 4 * MIB,
+    maxKeyBytes: 512,
+    maxMetadataEntries: 16,
+    maxMetadataBytes: 4096,
+    maxList: 1000,
+    maxUploads: 4,
+    ioBytes: 1024 * MIB,
+    minObjectBytes: 4096,
+    maxObjects: 100_000,
+};
+export const defaultAreas = {
+    short: { quotaBytes: 1024 * MIB, ttlMs: 24 * 3_600_000 },
+    long: { quotaBytes: 500 * MIB, ttlMs: 7 * 24 * 3_600_000 },
+    durable: { quotaBytes: 50 * MIB, ttlMs: null },
+};
+const cost = (o) => {
+    let overhead = Buffer.byteLength(o.key);
+    for (const [k, v] of Object.entries(o.metadata))
+        overhead += k.length + v.length;
+    return Math.max(o.size, localStorageLimits.minObjectBytes) + overhead;
+};
+const checkMatch = (ifMatch, current, key) => {
+    if (ifMatch === null)
+        return;
+    if (!current || (ifMatch !== "*" && ifMatch !== current.etag))
+        throw err("precondition_failed", `${key} changed or does not exist`);
+};
+export class StorageOpError extends Error {
+    kind;
+    code;
+    constructor(kind, code, message) {
+        super(message);
+        this.kind = kind;
+        this.code = code;
+    }
+}
+const err = (code, message, kind = "program") => new StorageOpError(kind, code, message);
+const BLOCKED_TYPES = /^(text\/html|application\/xhtml\+xml|text\/javascript|application\/javascript|application\/ecmascript|application\/x-sh|application\/x-msdownload|application\/x-executable|application\/x-mach-binary|application\/x-elf|image\/svg\+xml)$/;
+const BLOCKED_PREFIXES = [
+    [0x4d, 0x5a],
+    [0x7f, 0x45, 0x4c, 0x46],
+    [0xcf, 0xfa, 0xed, 0xfe],
+    [0x23, 0x21],
+];
+const startsWithMarkup = (bytes) => {
+    const head = Buffer.from(bytes.subarray(0, 64))
+        .toString("latin1")
+        .trimStart()
+        .toLowerCase();
+    return /^<(!doctype html|html|script|svg)/.test(head);
+};
+export class LocalStorage {
+    dir;
+    areas;
+    now;
+    shared;
+    #objects = new Map();
+    #bodies = new Map();
+    #uploads = new Map();
+    constructor(dir, areas = defaultAreas, now = Date.now, shared) {
+        this.dir = dir;
+        this.areas = areas;
+        this.now = now;
+        this.shared = shared;
+        for (const area of Object.keys(areas)) {
+            const index = new Map();
+            this.#objects.set(area, index);
+            if (!dir)
+                continue;
+            const folder = join(dir, area);
+            if (existsSync(folder))
+                for (const name of readdirSync(folder))
+                    if (name.endsWith(".tmp"))
+                        rmSync(join(folder, name), { force: true });
+            const file = join(folder, "index.json");
+            if (!existsSync(file))
+                continue;
+            let objects;
+            try {
+                objects = JSON.parse(readFileSync(file, "utf8"));
+            }
+            catch (e) {
+                throw new Error(`${file} is corrupt (${e.message}): delete the storage folder to start empty`);
+            }
+            for (const o of objects)
+                index.set(o.key, o);
+            const named = new Set(objects.map((o) => this.#fileName(o)));
+            for (const name of readdirSync(folder))
+                if (name !== "index.json" &&
+                    !named.has(name) &&
+                    !this.#legacy(name, index))
+                    rmSync(join(folder, name), { force: true });
+        }
+    }
+    #fileName = (o) => `${createHash("sha256").update(o.key).digest("hex")}.${o.etag}`;
+    #legacy(name, index) {
+        return (/^[0-9a-f]{64}$/.test(name) &&
+            [...index.values()].some((o) => this.#fileName(o).startsWith(`${name}.`)));
+    }
+    #area(area) {
+        const index = typeof area === "string" ? this.#objects.get(area) : undefined;
+        if (!index)
+            throw err("invalid_request", `Unknown storage area ${String(area)}`);
+        this.#expire(area, index);
+        return index;
+    }
+    #expire(area, index) {
+        const ttl = this.areas[area].ttlMs;
+        if (ttl === null)
+            return;
+        for (const [key, o] of index)
+            if (this.now() >= o.writtenAt + ttl)
+                this.#remove(area, index, key);
+    }
+    #bodyKey = (area, o) => `${area}/${this.#fileName(o)}`;
+    #body(area, o) {
+        const k = this.#bodyKey(area, o);
+        let body = this.#bodies.get(k);
+        if (!body && this.dir) {
+            const legacy = join(this.dir, area, createHash("sha256").update(o.key).digest("hex"));
+            try {
+                const file = join(this.dir, k);
+                body = new Uint8Array(readFileSync(existsSync(file) || !existsSync(legacy) ? file : legacy));
+            }
+            catch (e) {
+                throw err("unavailable", `The body of ${o.key} is missing from the storage folder (${e.code ?? "unreadable"})`, "infrastructure");
+            }
+            this.#bodies.set(k, body);
+        }
+        if (!body)
+            throw err("unavailable", "Object body lost", "infrastructure");
+        return body;
+    }
+    #drop(area, o) {
+        const k = this.#bodyKey(area, o);
+        this.#bodies.delete(k);
+        if (!this.dir)
+            return;
+        rmSync(join(this.dir, k), { force: true });
+        rmSync(join(this.dir, area, createHash("sha256").update(o.key).digest("hex")), {
+            force: true,
+        });
+    }
+    #remove(area, index, key) {
+        const o = index.get(key);
+        index.delete(key);
+        this.#save(area, index);
+        if (o)
+            this.#drop(area, o);
+    }
+    #save(area, index) {
+        if (!this.dir)
+            return;
+        mkdirSync(join(this.dir, area), { recursive: true });
+        const file = join(this.dir, area, "index.json");
+        writeFileSync(`${file}.tmp`, JSON.stringify([...index.values()]));
+        renameSync(`${file}.tmp`, file);
+    }
+    #expiresAt(area, o) {
+        const ttl = this.areas[area].ttlMs;
+        return ttl === null ? null : o.writtenAt + ttl;
+    }
+    #head(area, o) {
+        return {
+            key: o.key,
+            size: o.size,
+            contentType: o.contentType,
+            metadata: o.metadata,
+            lastModified: new Date(o.writtenAt).toISOString(),
+            writtenAt: o.writtenAt,
+            expiresAt: this.#expiresAt(area, o),
+            etag: o.etag,
+        };
+    }
+    #key(key) {
+        if (typeof key !== "string")
+            throw err("invalid_request", "A storage key is a string");
+        const bytes = Buffer.byteLength(key);
+        if (bytes < 1 || bytes > localStorageLimits.maxKeyBytes)
+            throw err("invalid_request", "A key holds 1 to 512 bytes of UTF-8");
+        if (/[\u0000-\u001f\u007f]/.test(key))
+            throw err("invalid_request", "A key has no control character");
+        return key;
+    }
+    #options(o, bytes) {
+        const options = (o ?? {});
+        if (options.ifMatch !== undefined && options.ifMatch !== null)
+            if (typeof options.ifMatch !== "string")
+                throw err("invalid_request", "ifMatch must be an etag string");
+        let type = typeof options.contentType === "string" && options.contentType
+            ? options.contentType
+            : "application/octet-stream";
+        type = type.split(";")[0].trim().toLowerCase();
+        if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(type))
+            throw err("invalid_request", "contentType is type/subtype");
+        if (BLOCKED_TYPES.test(type) ||
+            BLOCKED_PREFIXES.some((p) => p.every((b, i) => bytes[i] === b)) ||
+            startsWithMarkup(bytes))
+            throw err("blocked_type", "Executables, HTML and scripts are refused");
+        const metadata = (options.metadata ?? {});
+        const names = Object.keys(metadata);
+        if (names.length > localStorageLimits.maxMetadataEntries)
+            throw err("invalid_request", "At most 16 metadata entries");
+        let total = 0;
+        for (const name of names) {
+            const value = metadata[name];
+            if (!/^[a-z0-9-]{1,64}$/.test(name) ||
+                !/^[\x20-\x7e]{0,1024}$/.test(value))
+                throw err("invalid_request", `Invalid metadata ${name}`);
+            total += name.length + value.length;
+        }
+        if (total > localStorageLimits.maxMetadataBytes)
+            throw err("invalid_request", "Metadata over 4 KiB");
+        return {
+            contentType: type,
+            metadata: { ...metadata },
+            ifAbsent: !!options.ifAbsent,
+            ifMatch: typeof options.ifMatch === "string" ? options.ifMatch : null,
+        };
+    }
+    #used(area, index) {
+        let committed = 0;
+        for (const o of index.values())
+            committed += cost(o);
+        let reserved = 0;
+        for (const u of this.#uploads.values())
+            if (u.area === area)
+                reserved += u.size;
+        return { committed, reserved };
+    }
+    usedBytes(area) {
+        const index = this.#objects.get(area);
+        if (!index)
+            return 0;
+        const { committed, reserved } = this.#used(area, index);
+        return committed + reserved;
+    }
+    #quota(area, index) {
+        if (!this.shared) {
+            const { committed, reserved } = this.#used(area, index);
+            return {
+                quota: this.areas[area].quotaBytes,
+                used: committed + reserved,
+            };
+        }
+        let used = this.shared.outside();
+        for (const name of this.#objects.keys())
+            used += this.usedBytes(name);
+        return { quota: this.shared.quotaBytes, used };
+    }
+    copyFrom(source, from, to, keys) {
+        const src = source.#area(from);
+        const index = this.#area(to);
+        const selected = (keys ?? [...src.keys()].sort())
+            .map((k) => src.get(k))
+            .filter((o) => o !== undefined);
+        let needed = 0;
+        let credit = 0;
+        let fresh = 0;
+        for (const o of selected) {
+            needed += cost(o);
+            const old = index.get(o.key);
+            if (old)
+                credit += cost(old);
+            else
+                fresh++;
+        }
+        const { quota, used } = this.#quota(to, index);
+        if (used - credit + needed > quota)
+            throw err("quota_exceeded", `The ${to} area is full`, "limit");
+        if (index.size + fresh > localStorageLimits.maxObjects)
+            throw err("quota_exceeded", `The ${to} area holds too many objects`, "limit");
+        let bytes = 0;
+        for (const o of selected) {
+            const body = source.#body(from, o);
+            this.#write(to, o.key, body, {
+                contentType: o.contentType,
+                metadata: o.metadata,
+                ifAbsent: false,
+                ifMatch: null,
+            });
+            bytes += o.size;
+        }
+        return { objects: selected.length, bytes };
+    }
+    #write(area, key, body, o, pending = 0) {
+        const index = this.#area(area);
+        if (body.length > localStorageLimits.maxObjectBytes)
+            throw err("too_large", "An object holds at most 64 MiB", "limit");
+        const previous = index.get(key);
+        if (previous && o.ifAbsent)
+            throw err("already_exists", `${key} exists`);
+        checkMatch(o.ifMatch, previous, key);
+        if (!previous && index.size >= localStorageLimits.maxObjects)
+            throw err("quota_exceeded", `The ${area} area holds too many objects`, "limit");
+        const quota = this.#quota(area, index);
+        const used = quota.used - pending - (previous ? cost(previous) : 0);
+        const needed = cost({ key, size: body.length, metadata: o.metadata });
+        if (used + needed > quota.quota)
+            throw err("quota_exceeded", `The ${area} area is full`, "limit");
+        const stored = {
+            key,
+            size: body.length,
+            contentType: o.contentType,
+            metadata: o.metadata,
+            writtenAt: this.now(),
+            etag: randomBytes(8).toString("hex"),
+        };
+        const k = this.#bodyKey(area, stored);
+        this.#bodies.set(k, body);
+        if (this.dir) {
+            mkdirSync(join(this.dir, area), { recursive: true });
+            writeFileSync(`${join(this.dir, k)}.tmp`, body);
+            renameSync(`${join(this.dir, k)}.tmp`, join(this.dir, k));
+        }
+        index.set(key, stored);
+        this.#save(area, index);
+        if (previous)
+            this.#drop(area, previous);
+        return this.#head(area, stored);
+    }
+    endExecution(owner) {
+        for (const [id, u] of this.#uploads)
+            if (u.owner === owner)
+                this.#uploads.delete(id);
+    }
+    op(name, a, owner, io) {
+        const charge = (n) => {
+            io.bytes += n;
+            if (io.bytes > localStorageLimits.ioBytes)
+                throw err("io_limit", "Over 1 GiB of storage read and written", "limit");
+        };
+        const decode = (data) => {
+            if (typeof data !== "string")
+                throw err("invalid_request", "data is base64");
+            const bytes = new Uint8Array(Buffer.from(data, "base64"));
+            if (bytes.length > localStorageLimits.chunkBytes)
+                throw err("invalid_request", "A chunk holds at most 4 MiB");
+            return bytes;
+        };
+        switch (name) {
+            case "StorageGet": {
+                const area = a.area;
+                const index = this.#area(area);
+                const o = index.get(this.#key(a.key));
+                if (!o)
+                    return null;
+                const offset = Number(a.offset ?? 0);
+                const length = Math.min(Number(a.length ?? localStorageLimits.chunkBytes), localStorageLimits.chunkBytes);
+                if (!Number.isSafeInteger(offset) || offset < 0 || !(length >= 0))
+                    throw err("invalid_request", "Invalid range");
+                const body = this.#body(area, o);
+                const start = Math.min(offset, body.length);
+                const chunk = body.subarray(start, Math.min(body.length, start + length));
+                charge(chunk.length);
+                return {
+                    ...this.#head(area, o),
+                    offset: start,
+                    data: Buffer.from(chunk).toString("base64"),
+                };
+            }
+            case "StorageHead": {
+                const area = a.area;
+                const o = this.#area(area).get(this.#key(a.key));
+                return o ? this.#head(area, o) : null;
+            }
+            case "StoragePut": {
+                const area = a.area;
+                const key = this.#key(a.key);
+                const body = decode(a.data);
+                charge(body.length);
+                return this.#write(area, key, body, this.#options(a.options, body));
+            }
+            case "StorageUploadStart": {
+                const area = a.area;
+                this.#area(area);
+                const key = this.#key(a.key);
+                let open = 0;
+                for (const u of this.#uploads.values())
+                    if (u.owner === owner)
+                        open++;
+                if (open >= localStorageLimits.maxUploads)
+                    throw err("upload_limit", "At most 4 writer() open", "limit");
+                const o = this.#options(a.options, new Uint8Array(0));
+                const id = randomBytes(8).toString("hex");
+                this.#uploads.set(id, { area, key, ...o, chunks: [], size: 0, owner });
+                return id;
+            }
+            case "StorageUploadChunk": {
+                const body = decode(a.data);
+                const id = a.upload;
+                const u = this.#uploads.get(id);
+                if (!u)
+                    throw err("invalid_request", "Unknown upload");
+                this.#uploads.delete(id);
+                charge(body.length);
+                if (u.size + body.length > localStorageLimits.maxObjectBytes)
+                    throw err("too_large", "An object holds at most 64 MiB", "limit");
+                const index = this.#area(u.area);
+                const quota = this.#quota(u.area, index);
+                const previous = index.get(u.key)?.size ?? 0;
+                if (quota.used - previous + body.length > quota.quota)
+                    throw err("quota_exceeded", `The ${u.area} area is full`, "limit");
+                u.chunks.push(body);
+                u.size += body.length;
+                this.#uploads.set(id, u);
+                return null;
+            }
+            case "StorageUploadCommit": {
+                const id = a.upload;
+                const u = this.#uploads.get(id);
+                if (!u)
+                    throw err("invalid_request", "Unknown upload");
+                const joined = Buffer.concat(u.chunks);
+                const body = new Uint8Array(joined.buffer, joined.byteOffset, joined.length);
+                this.#options({ contentType: u.contentType }, body);
+                const head = this.#write(u.area, u.key, body, u, u.size);
+                this.#uploads.delete(id);
+                return head;
+            }
+            case "StorageUploadAbort":
+                this.#uploads.delete(a.upload);
+                return null;
+            case "StorageDelete": {
+                const area = a.area;
+                const index = this.#area(area);
+                const key = this.#key(a.key);
+                const ifMatch = a.if_match ?? null;
+                if (ifMatch !== null && typeof ifMatch !== "string")
+                    throw err("invalid_request", "ifMatch must be an etag string");
+                checkMatch(ifMatch, index.get(key), key);
+                if (index.has(key))
+                    this.#remove(area, index, key);
+                return null;
+            }
+            case "StorageList": {
+                const area = a.area;
+                const index = this.#area(area);
+                const prefix = typeof a.prefix === "string" ? a.prefix : "";
+                const after = typeof a.after === "string" ? a.after : null;
+                const max = Math.min(Number(a.max_objects ?? localStorageLimits.maxList), localStorageLimits.maxList);
+                const keys = [...index.keys()]
+                    .filter((k) => k.startsWith(prefix) &&
+                    (after === null ||
+                        Buffer.compare(Buffer.from(k), Buffer.from(after)) > 0))
+                    .sort((x, y) => Buffer.compare(Buffer.from(x), Buffer.from(y)));
+                const page = keys.slice(0, max);
+                return {
+                    objects: page.map((k) => {
+                        const o = index.get(k);
+                        const h = this.#head(area, o);
+                        return {
+                            key: h.key,
+                            size: h.size,
+                            contentType: h.contentType,
+                            lastModified: h.lastModified,
+                            expiresAt: h.expiresAt,
+                        };
+                    }),
+                    cursor: keys.length > max ? (page[page.length - 1] ?? null) : null,
+                };
+            }
+            case "StorageUsage": {
+                const area = a.area;
+                const index = this.#area(area);
+                const { committed, reserved } = this.#used(area, index);
+                return {
+                    area,
+                    initialized: true,
+                    usedBytes: committed + reserved,
+                    reservedBytes: reserved,
+                    uncertainBytes: 0,
+                    committedBytes: committed,
+                    deletingBytes: 0,
+                    objects: index.size,
+                    reservations: [...this.#uploads.values()]
+                        .filter((u) => u.area === area)
+                        .map((u) => ({
+                        key: u.key,
+                        size: u.size,
+                        state: "writing",
+                        updatedAt: new Date(this.now()).toISOString(),
+                    })),
+                    quotaBytes: this.shared?.quotaBytes ?? this.areas[area].quotaBytes,
+                    ttlMs: this.areas[area].ttlMs,
+                    maxObjectBytes: localStorageLimits.maxObjectBytes,
+                };
+            }
+            default:
+                throw err("invalid_request", `Unknown storage operation ${name}`);
+        }
+    }
+}