@vertesia/memory 0.24.0-dev.202601221707

package/src/utils/rewrite.ts

@@ -0,0 +1,167 @@
+import { basename, dirname, extname, join } from "path";
+
+/**
+ * The path argument is the empty string when mapping streams or buffers not related to a file system file.
+ */
+export type PathMapperFn = ((path: string, index: number) => string);
+
+export function createPathRewrite(path: string): PathMapperFn {
+    let truncPath: (path: string) => string;
+    let basePath: string = '';
+    let index = path.indexOf('!');
+    if (index > -1) {
+        basePath = path.substring(0, index);
+        if (!basePath.endsWith('/')) {
+            basePath += '/';
+        }
+        truncPath = (path: string) => {
+            return path.substring(basePath.length);
+        }
+        path = path.substring(index + 1);
+    } else {
+        truncPath = (path: string) => {
+            return basename(path);
+        }
+    }
+    if (path === '*') {
+        // preserve path
+        return truncPath;
+    } else if (path.endsWith("/*")) {
+        const prefix = path.slice(0, -2);
+        return (path: string) => {
+            path = truncPath(path);
+            return join(prefix, path);
+        }
+    } else {
+        // use path builder
+        return buildPathRewrite(path, truncPath);
+    }
+}
+
+const RX_PARTS = /(%d\/)|(\.?%e)|(%[fnip])/g;
+function buildPathRewrite(path: string, truncPath: (path: string) => string): PathMapperFn {
+    let parts: ((path: Path, index: number) => string)[] = [];
+    let m: RegExpExecArray | null;
+    let lastIndex = 0;
+    while (m = RX_PARTS.exec(path)) {
+        if (m.index > lastIndex) {
+            const literal = path.substring(lastIndex, m.index);
+            parts.push(() => literal);
+        }
+        if (m[1]) { // %d/
+            parts.push((path: Path) => path.dirname ? path.dirname + '/' : '');
+        } else if (m[2]) { // .?%e
+            if (m[2][0] === '.') {
+                parts.push((path: Path) => path.extname || '');
+            } else {
+                parts.push((path: Path) => path.extname ? path.extname.slice(1) : ''); // extension without dot
+            }
+        } else if (m[3]) {
+            switch (m[3]) {
+                case '%f':
+                    parts.push((path: Path) => path.name);
+                    break;
+                case '%n':
+                    parts.push((path: Path) => path.basename);
+                    break;
+                case '%p': // stringify the path by replacing / with _
+                    parts.push((path: Path) => {
+                        let p = path.value;
+                        if (p.startsWith('/')) {
+                            p = p.substring(1);
+                        }
+                        return p.replaceAll('/', '_');
+                    });
+                    break;
+                case '%i': // index
+                    parts.push((_path: Path, index: number) => String(index));
+                    break;
+                default: throw new Error(`Bug: should never happen`);
+            }
+        }
+        lastIndex = m.index + m[0].length;
+    }
+    if (!parts.length) {
+        return () => path;
+    } else {
+        if (lastIndex < path.length) {
+            const literal = path.substring(lastIndex);
+            parts.push(() => literal);
+        }
+        return (path: string, index: number) => {
+            const pathObj = new Path(truncPath(path));
+            const out = [];
+            for (const part of parts) {
+                out.push(part(pathObj, index));
+            }
+            return out.join('');
+        }
+    }
+}
+
+
+export class Path {
+    _name?: string;
+    _extname?: string;
+    _dirname?: string;
+    _basename?: string;
+
+    /**
+     * The complete path value
+     */
+    value: string
+
+    /**
+     * The file name (the last portion of the path). Includes the extension if present.
+     */
+    get name(): string {
+        if (!this._name) {
+            this._name = basename(this.value);
+        }
+        return this._name;
+    }
+
+    /**
+     * The extension of the file  including the leading '.'.
+     * An empty string if the file has no extension.
+     */
+    get extname(): string {
+        if (!this._extname) {
+            this._extname = extname(this.value);
+        }
+        return this._extname;
+    }
+    /**
+     * The directory portion of the path. Doesn't include the trailing slash.
+     * If no directory is present, returns an empty string.
+     */
+    get dirname(): string {
+        if (!this._dirname) {
+            this._dirname = dirname(this.value);
+            if (this._dirname === '.') {
+                this._dirname = '';
+            }
+        }
+        return this._dirname;
+    }
+    /**
+     * The path without the extension
+     */
+    get basename(): string {
+        if (!this._basename) {
+            this._basename = this.extname ? this.name.slice(0, -this.extname.length) : this.name;
+        }
+        return this._basename;
+    }
+
+    constructor(value: string) {
+        this.value = value;
+    }
+
+    /**
+     * Return the complete path value (same as `value`)
+     */
+    toString(): string {
+        return this.value;
+    }
+}

package/src/utils/stream.test.ts

@@ -0,0 +1,13 @@
+import { describe, test, expect } from "vitest";
+import { BufferWritableStream } from "./stream";
+
+describe("BufferWritableStream", () => {
+    test("write buffer", () => {
+        const stream = new BufferWritableStream();
+        stream.write(Buffer.from("hello"));
+        stream.write(Buffer.from(" "));
+        stream.write(Buffer.from("world"));
+        stream.end();
+        expect(stream.getText()).toBe("hello world");
+    })
+})

package/src/utils/stream.ts

@@ -0,0 +1,27 @@
+import { Writable } from "stream";
+
+export class BufferWritableStream extends Writable {
+    chunks: Buffer[] = []
+    buffer: Buffer | undefined;
+
+    // _write method is required to handle the incoming data
+    _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+        this.chunks.push(chunk); // Collect the chunk into the array
+        callback(); // Indicate that the write is complete
+    }
+
+    // Optional _final method is called when the stream is ending
+    _final(callback: (error?: Error | null) => void) {
+        this.buffer = Buffer.concat(this.chunks); // Concatenate the collected chunks into a buffer
+        callback(); // Indicate the stream is finished
+    }
+
+    // Method to get the final buffer when the stream is closed
+    getBuffer() {
+        return this.buffer;
+    }
+
+    getText(encoding: BufferEncoding = "utf-8") {
+        return this.buffer?.toString(encoding);
+    }
+}

package/src/utils/tar.test.ts

@@ -0,0 +1,48 @@
+import { describe, test, expect, beforeAll, afterAll } from "vitest";
+import { loadTarIndex, TarBuilder, TarIndex } from "./tar";
+import { readFileSync, stat, statSync, unlinkSync } from "fs";
+
+const tarFile = `test-${Date.now()}.tar`;
+afterAll(() => {
+    unlinkSync(tarFile);
+});
+
+describe("Indexed tar format", () => {
+    const builder = new TarBuilder(tarFile);
+    test("build tar", async () => {
+        builder.add("file1.txt", Buffer.from("hello world!"));
+        builder.add("file2.txt", Buffer.from("bonjour monde!"));
+        builder.add("app/package.json", readFileSync("./package.json"));
+        await builder.build();
+        const stats = statSync(tarFile);
+        expect(stats.isFile()).toBeTruthy();
+    });
+
+    test("read tar", async () => {
+        const index = await loadTarIndex(tarFile) as TarIndex;
+        expect(index).toBeDefined();
+        expect(Object.keys(index.entries).length).toBe(3);
+        const file1 = index.get('file1.txt');
+        const file2 = index.get('file2.txt');
+        const file3 = index.get('app/package.json');
+        expect(file1).toBeDefined();
+        expect(file2).toBeDefined();
+        expect(file3).toBeDefined();
+        expect(file1!.size).toBe(12);
+        expect(file2!.size).toBe(14);
+        expect(file3!.size).toBeGreaterThan(0);
+
+        const content1 = await index.getContent('file1.txt');
+        const content2 = await index.getContent('file2.txt');
+        const content3 = await index.getContent('app/package.json');
+
+        expect(content1!.toString()).toBe("hello world!");
+        expect(content2!.toString()).toBe("bonjour monde!");
+        const pkg = JSON.parse(content3!.toString());
+        expect(pkg).toBeDefined();
+        expect(pkg).toHaveProperty("name");
+        expect(pkg).toHaveProperty("version");
+        expect(pkg).toHaveProperty("dependencies");
+    });
+
+});

package/src/utils/tar.ts

@@ -0,0 +1,203 @@
+import fs from "fs";
+import { FileHandle, open } from "fs/promises";
+import { pipeline } from "stream/promises";
+import tar from "tar-stream";
+import zlib from "zlib";
+
+export interface TarEntry {
+    name: string;
+    getContent(): Promise<Buffer>;
+}
+
+export class TarBuilder {
+    pack: tar.Pack;
+    indexData: string[] = [];
+    currentOffset = 0;
+    tarPromise: Promise<unknown>;
+
+    constructor(file: string) {
+        const pack = tar.pack(); // Create a new tar stream
+        this.pack = pack;
+        // Open the output file as a write stream
+        const outputStream = fs.createWriteStream(file);
+        if (file.endsWith('.gz')) {
+            this.tarPromise = pipeline(pack, zlib.createGzip(), outputStream);
+        } else {
+            this.tarPromise = pipeline(pack, outputStream);
+        }
+    }
+
+
+    async add(name: string, content?: Buffer) {
+        name = normalizePath(name);
+        // Calculate header size, 512 bytes for tar headers
+        const headerSize = 512;
+        const contentSize = content ? Buffer.byteLength(content) : 0;
+        const entryHeaderOffset = this.currentOffset;
+
+        // Store the index entry
+        // entry data offset is always at header offset + 512 bytes
+        if (contentSize > 0) { // do not index directories
+            this.indexData.push(`${name}:${entryHeaderOffset},${contentSize}`);
+        }
+
+        // Add the file entry to the tar stream
+        this.pack.entry({ name, size: contentSize }, content);
+
+        // Update the offset
+        this.currentOffset += headerSize + contentSize;
+        // Tar files are padded to 512-byte boundaries
+        if (contentSize % 512 !== 0) {
+            this.currentOffset += 512 - (contentSize % 512);
+        }
+    }
+
+    async build() {
+        const pack = this.pack;
+        // Convert index data to string and calculate its size
+        const indexContent = this.indexData.join('\n') + '\n';
+        const indexContentSize = Buffer.byteLength(indexContent);
+
+        // Add the .index entry to the tar
+        pack.entry({ name: '.index', size: indexContentSize }, indexContent);
+
+        pack.finalize(); // Finalize the tar stream
+
+        await this.tarPromise;
+    }
+
+    destroy() {
+        this.pack.destroy();
+    }
+
+}
+
+export async function loadTarIndex(tarFile: string) {
+    const fd = await open(tarFile, 'r');
+    try {
+        return await readTarIndex(fd);
+    } catch (err) {
+        await fd.close();
+        throw err;
+    }
+}
+
+async function readTarIndex(fd: FileHandle) {
+    const stats = await fd.stat();
+    const size = stats.size;
+    // we want to find the index header.
+    // we read the last chunks of 512 until we find the file name followed by a 0 char.
+    // the tar file ends with a segment of 1024 bytes of 0 so we need to skip that.
+    // we pick a size for the buffer to also include the file size entry from the header. So the buffer should be
+    // of 100 + 8 + 8 + 8 + 12 bytes = 124 + 12 bytes = 136 bytes
+    // the file size will be located at offset 124 and is 12 bytes long
+    // skip 1024 0 bytes then skip another 1024 bytes to find the first possible location of the index header (512 bytes for content and 512 bytes for the header)
+    let offset = size - 1024 - 1024;
+    const buffer = Buffer.alloc(512);
+    while (offset >= 0) {
+        await fd.read(buffer, 0, 512, offset);
+        // remove the 0 byte padding
+        const fileName = buffer.toString('utf-8', 0, 100);
+        if (fileName.startsWith('.index\0')) {
+            // we found the index header
+            const indexSize = getHeaderFileSize(buffer);
+            const indexDataOffset = offset + 512;
+            const indexDataEnd = indexDataOffset + indexSize;
+            if (indexDataEnd > size - 1024) {
+                throw new Error('Invalid index data offsets: [' + indexDataOffset + ':' + indexDataEnd + ']');
+            }
+            const dataBuffer = Buffer.alloc(indexSize);
+            await fd.read(dataBuffer, 0, indexSize, indexDataOffset);
+            const indexContent = dataBuffer.toString('utf-8');
+            return new TarIndex(fd, indexContent);
+        }
+        offset -= 512;
+    }
+    return null;
+}
+
+export interface TarEntryIndex {
+    offset: number,
+    size: number
+}
+export class TarIndex {
+    entries: Record<string, TarEntryIndex> = {};
+    headerBuffer = Buffer.alloc(512);
+    /**
+     * @param fd the tar file descriptor
+     * @param content the index content
+     */
+    constructor(public fd: FileHandle, content: string) {
+        const lines = content.split('\n');
+        for (const line of lines) {
+            if (line) {
+                const [name, value] = line.split(':');
+                const [offsetStr, sizeStr] = value.split(',');
+                const offset = parseInt(offsetStr);
+                const size = parseInt(sizeStr);
+                this.entries[name] = { offset, size };
+            }
+        }
+    }
+
+    getPaths() {
+        return Object.keys(this.entries);
+    }
+
+    getSortedPaths() {
+        return Object.keys(this.entries).sort();
+    }
+
+    get(name: string) {
+        return this.entries[name];
+    }
+
+    async getContentAt(offset: number, size: number) {
+        const buffer = Buffer.alloc(size);
+        await this.fd.read(buffer, 0, size, offset + 512);
+        return buffer;
+    }
+    async getContent(name: string) {
+        const entry = this.entries[name];
+        if (entry) {
+            return this.getContentAt(entry.offset, entry.size);
+        } else {
+            return null;
+        }
+    }
+
+    getReadStream(name: string, encoding?: BufferEncoding) {
+        const entry = this.entries[name];
+        if (entry) {
+            const offset = entry.offset + 512;
+            return this.fd.createReadStream({
+                encoding,
+                start: entry.offset,
+                end: offset + entry.size
+            })
+        } else {
+            return null;
+        }
+    }
+
+    async close() {
+        await this.fd.close();
+    }
+
+}
+
+
+function getHeaderFileSize(buffer: Buffer) {
+    const octalSize = buffer.toString('ascii', 124, 136).trim();
+    return parseInt(octalSize, 8);
+}
+
+export function normalizePath(path: string) {
+    if (path.startsWith('/')) {
+        path = path.slice(1);
+    }
+    if (path.endsWith('/')) {
+        path = path.slice(-1);
+    }
+    return path;
+}