@vertesia/memory 0.43.0

package/src/commands/copy.ts

@@ -0,0 +1,53 @@
+import { Builder } from "../Builder.js";
+import { DocxObject, MediaObject, MediaOptions, PdfObject } from "../ContentObject.js";
+import { AbstractContentSource, ContentSource, FileSource, SourceSpec } from "../ContentSource.js";
+import { createPathRewrite, PathMapperFn } from "../utils/rewrite.js";
+
+function rewritePath(source: ContentSource, index: number, mapper: PathMapperFn) {
+    const path = source instanceof FileSource ? source.file : '';
+    return mapper(path, index);
+}
+
+export interface CopyOptions {
+    media?: MediaOptions;
+    extractText?: boolean | string;
+}
+export function copy(builder: Builder, source: SourceSpec, toPath: string, options: CopyOptions = {}) {
+    const resolved = AbstractContentSource.resolve(source);
+    const mapperFn = createPathRewrite(toPath);
+    if (Array.isArray(resolved)) {
+        for (let i = 0, l = resolved.length; i < l; i++) {
+            const cs = resolved[i];
+            copyOne(builder, cs, rewritePath(cs, i, mapperFn), options);
+        }
+    } else {
+        return copyOne(builder, resolved, rewritePath(resolved, 0, mapperFn), options);
+    }
+}
+
+function copyOne(builder: Builder, source: ContentSource, toPath: string, options: CopyOptions) {
+    if (options.media) {
+        new MediaObject(builder, source, options.media).copyTo(toPath);
+    } else if (options.extractText) {
+        let type = options.extractText;
+        if (typeof type === "boolean") {
+            if (source instanceof FileSource) {
+                type = source.extname ? source.extname.slice(1) : source.extname;
+            } else {
+                throw new Error("source type for extractText must be spefcified");
+            }
+        }
+        switch (type) {
+            case "pdf":
+                new PdfObject(builder, source).copyTo(toPath);
+                break;
+            case "docx":
+                new DocxObject(builder, source).copyTo(toPath);
+                break;
+            default:
+                throw new Error("Unsupported extractText type: " + type);
+        }
+    } else {
+        builder.addEntry(toPath, source);
+    }
+}

package/src/commands/exec.test.ts

@@ -0,0 +1,22 @@
+import { describe, test, expect } from "vitest";
+import { exec } from "./exec";
+import fs from "fs";;
+
+describe("exec", () => {
+    test("test pipe", async () => {
+        const output = await exec('echo "hello" | wc -c', { quiet: true });
+        expect(output).toBeDefined();
+        expect(output!.trim()).toBe("6");
+    });
+    test("test pipe with redirection", async () => {
+        const name = "test-" + Date.now().toString() + ".txt";
+        const output = await exec(`echo "hello" | wc -c > ${name}`, { quiet: true });
+        try {
+            expect(output).toBeUndefined();
+            const content = fs.readFileSync(name, 'utf-8').trim();
+            expect(content).toBe("6");
+        } finally {
+            fs.unlinkSync(name);
+        }
+    });
+});

package/src/commands/exec.ts

@@ -0,0 +1,83 @@
+import { ChildProcess, spawn } from 'child_process';
+import fs from 'fs';
+import { Stream, Writable } from 'stream';
+import { Command, splitPipeCommands } from '../utils/cmdline.js';
+import { BufferWritableStream } from '../utils/stream.js';
+
+export interface ExecOptions {
+    quiet?: boolean;
+}
+
+export async function exec(commandLine: string, options: ExecOptions = {}): Promise<string | undefined> {
+    const verbose = !options.quiet;
+    commandLine = commandLine.trim();
+    const { commands, out } = splitPipeCommands(commandLine);
+    if (!commands.length) {
+        throw new Error('Invalid command line. No command: ' + commandLine);
+    }
+
+    let outStream: Writable;
+    if (out) {
+        outStream = fs.createWriteStream(out, { flags: 'w' });
+    } else {
+        outStream = new BufferWritableStream();
+    }
+    const pipePromise = executePipe(commands, outStream, verbose);
+    const outPromise = new Promise((resolve, reject) => {
+        outStream.on('finish', () => {
+            resolve(undefined);
+        });
+        outStream.on('error', (err: Error) => {
+            reject(err);
+            outStream.destroy();
+        });
+    });
+
+    const [status] = await Promise.all([pipePromise, outPromise]);
+
+    if (verbose) {
+        if (!status) {
+            console.log(`Command: ${commandLine} exited with status ${status}`);
+        } else {
+            console.error(`Command: ${commandLine} exited with status ${status}`);
+        }
+    }
+
+    if (outStream instanceof BufferWritableStream) {
+        return outStream.getText();
+    } else {
+        return undefined;
+    }
+}
+
+
+export function executePipe(commands: Command[], finalOutput: Writable | undefined, verbose: boolean = false) {
+    return new Promise((resolve, reject) => {
+        let input: Stream | undefined;
+        let child: ChildProcess | undefined;
+        for (const cmd of commands) {
+            verbose && console.log(`Running: ${cmd.name} ${cmd.args?.join(' ')}`);
+            child = spawn(cmd.name, cmd.args, {
+                // in, out, err
+                stdio: ['pipe', 'pipe', 'inherit'],
+            });
+            if (input) {
+                input.pipe(child.stdin!);
+            }
+            input = child.stdout!;
+            child.on("error", (err: Error) => {
+                console.error(`Failed to run ${cmd.name}`, err);
+                reject(err);
+            })
+        }
+        if (child) {
+            child.on("exit", (code: number | null, signal: string | null) => {
+                resolve(code !== null ? code : signal);
+            });
+            finalOutput && child.stdout?.pipe(finalOutput);
+        } else {
+            reject(new Error("no child spawned"));
+        }
+    });
+
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export { Builder, buildMemoryPack } from "./Builder.js"
+export type { BuildOptions, Commands } from "./Builder.js"
+export type { CopyOptions } from "./commands/copy.js"
+export type { ExecOptions } from "./commands/exec.js"
+export { ContentObject, DocxObject, JsonObject, MediaObject, PdfObject } from "./ContentObject.js"
+export type { MediaOptions } from "./ContentObject.js"
+export { AbstractContentSource, BufferSource, FileSource, TextSource } from "./ContentSource.js"
+export type { ContentSource, SourceSpec } from "./ContentSource.js"
+export { MemoryEntry, TarMemoryPack, loadMemoryPack } from "./MemoryPack.js"
+export type { MemoryPack, ProjectionProperties } from "./MemoryPack.js"
+export { MemoryPackBuilder } from "./MemoryPackBuilder.js"
+export type { FromOptions } from "./MemoryPackBuilder.js"
+export { TarBuilder, TarIndex, loadTarIndex } from './utils/tar.js'
+export type { TarEntry, TarEntryIndex } from './utils/tar.js'

package/src/utils/cmdline.test.ts

@@ -0,0 +1,32 @@
+import { describe, test, expect } from "vitest";
+import { splitCommandLine, splitPipeCommands } from "./cmdline";
+
+describe("command line parser", () => {
+
+    test("split command line", () => {
+        const args = splitCommandLine("cmd -m \"hello 'world'\" 'some \"file\"' 'nested \"dquote\"' \"nested \'squote\'\"");
+        expect(args).toEqual(["cmd", "-m", "hello 'world'", "some \"file\"", 'nested "dquote"', "nested 'squote'"]);
+
+    })
+
+    test("split commands pipeline", () => {
+        const pipe = splitPipeCommands("cat \"the file.txt\" | grep \"the\" | wc -c");
+        expect(pipe.out).toBeUndefined();
+        expect(pipe.commands).toEqual([
+            { name: "cat", args: ["the file.txt"] },
+            { name: "grep", args: ["the"] },
+            { name: "wc", args: ["-c"] }
+        ]);
+    })
+
+    test("split commands pipeline and redirect output", () => {
+        const pipe = splitPipeCommands("cat \"the file.txt\" | grep \"the\" | wc -c > out.txt");
+        expect(pipe.out).toBe("out.txt");
+        expect(pipe.commands).toEqual([
+            { name: "cat", args: ["the file.txt"] },
+            { name: "grep", args: ["the"] },
+            { name: "wc", args: ["-c"] }
+        ]);
+    })
+
+});

package/src/utils/cmdline.ts

@@ -0,0 +1,92 @@
+
+function readWord(text: string, index: number): [string, number] {
+    let i = index, len = text.length;
+    while (i < len && (text[i] !== ' ' || text[i] !== '\t')) {
+        const c = text[i];
+        if (c === ' ' || c === '\t') {
+            return [text.substring(index, i), i];
+        }
+        i++;
+    }
+    return [text.substring(index), len];
+}
+
+function readQuotedArg(text: string, index: number, quote: string): [string, number] {
+    let i = index, len = text.length;
+    while (i < len) {
+        const c = text[i];
+        if (c === '\\') {
+            i += 2;
+            continue;
+        } else if (c === quote) {
+            return [text.substring(index, i), i + 1];
+        } else {
+            i++;
+        }
+    }
+    return [text.substring(index), len];
+}
+
+export function splitCommandLine(text: string) {
+    const args = [];
+    let i = 0, len = text.length;
+    while (i < len) {
+        let c = text[i];
+        if (c === ' ' || c === '\t') {
+            i++; // skip whitespace
+        } else if (c === '"' || c === "'") {
+            const [word, offset] = readQuotedArg(text, i + 1, c);
+            i = offset;
+            args.push(word);
+        } else {
+            const [word, offset] = readWord(text, i);
+            i = offset;
+            args.push(word);
+        }
+    }
+    return args;
+}
+
+export interface Command {
+    name: string;
+    args: string[];
+}
+export interface CommandPipe {
+    commands: Command[];
+    out?: string | undefined;
+}
+export function splitPipeCommands(text: string): CommandPipe {
+    const tokens = splitCommandLine(text);
+    const commands: Command[] = [];
+    let args: string[] = [];
+    for (const token of tokens) {
+        if (token === "|") {
+            if (args.length < 1) {
+                throw new Error("Invalid pipe character. Expecting a command first.");
+            }
+            const name = args.shift()!;
+            commands.push({
+                name,
+                args
+            });
+            args = [];
+        } else {
+            args.push(token);
+        }
+    }
+    let out: string | undefined;
+    if (args.length > 0) {
+        const name = args.shift()!;
+        if (args.length > 1 && args[args.length - 2] === ">") {
+            out = args.pop();
+            args.pop(); // remove the ">"
+        }
+        commands.push({
+            name,
+            args
+        });
+    }
+    return {
+        commands, out
+    };
+}

package/src/utils/rewrite.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, test } from "vitest";
+import { createPathRewrite } from "./rewrite";
+
+describe("Path rewrite", () => {
+    test("constant path", () => {
+        let fn = createPathRewrite("/root!some/path")
+        let r = fn("/root/the/path", 0)
+        expect(r).toBe("some/path")
+
+        fn = createPathRewrite("some/path")
+        r = fn("/root/the/path", 0)
+        expect(r).toBe("some/path")
+    })
+
+    test("wildcard path", () => {
+        let fn = createPathRewrite("/root!*")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("the/path.ext")
+
+        fn = createPathRewrite("*")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("path.ext")
+    })
+
+    test("expression path", () => {
+        let fn = createPathRewrite("/root!images/%d/file-%n-%i.%e")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the/file-path-0.ext")
+
+        fn = createPathRewrite("images/%d/file-%n-%i.%e")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/file-path-0.ext")
+    })
+
+    test("expression path using %f", () => {
+        let fn = createPathRewrite("/root!images/%d/file-%i-%f")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the/file-0-path.ext")
+
+        fn = createPathRewrite("images/%d/file-%i-%f")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/file-0-path.ext")
+    })
+
+    test("expression using widlcard with prefix", () => {
+        let fn = createPathRewrite("/root!images/*")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the/path.ext")
+
+        fn = createPathRewrite("images/*")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/path.ext")
+    })
+
+    test("expression using %p", () => {
+        let fn = createPathRewrite("/root!images/%p")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the_path.ext")
+
+        fn = createPathRewrite("images/%p")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/path.ext")
+    })
+
+})

package/src/utils/rewrite.ts

@@ -0,0 +1,167 @@
+import { basename, dirname, extname, join } from "path";
+
+/**
+ * The path argumentmay is the empty string when mapping streams or buffers not related to a file system file.
+ */
+export type PathMapperFn = ((path: string, index: number) => string);
+
+export function createPathRewrite(path: string): PathMapperFn {
+    let truncPath: (path: string) => string;
+    let basePath: string = '';
+    let index = path.indexOf('!');
+    if (index > -1) {
+        basePath = path.substring(0, index);
+        if (!basePath.endsWith('/')) {
+            basePath += '/';
+        }
+        truncPath = (path: string) => {
+            return path.substring(basePath.length);
+        }
+        path = path.substring(index + 1);
+    } else {
+        truncPath = (path: string) => {
+            return basename(path);
+        }
+    }
+    if (path === '*') {
+        // preserve path
+        return truncPath;
+    } else if (path.endsWith("/*")) {
+        const prefix = path.slice(0, -2);
+        return (path: string) => {
+            path = truncPath(path);
+            return join(prefix, path);
+        }
+    } else {
+        // use path builder
+        return buildPathRewrite(path, truncPath);
+    }
+}
+
+const RX_PARTS = /(%d\/)|(\.?%e)|(%[fnip])/g;
+function buildPathRewrite(path: string, truncPath: (path: string) => string): PathMapperFn {
+    let parts: ((path: Path, index: number) => string)[] = [];
+    let m: RegExpExecArray | null;
+    let lastIndex = 0;
+    while (m = RX_PARTS.exec(path)) {
+        if (m.index > lastIndex) {
+            const literal = path.substring(lastIndex, m.index);
+            parts.push(() => literal);
+        }
+        if (m[1]) { // %d/
+            parts.push((path: Path) => path.dirname ? path.dirname + '/' : '');
+        } else if (m[2]) { // .?%e
+            if (m[2][0] === '.') {
+                parts.push((path: Path) => path.extname || '');
+            } else {
+                parts.push((path: Path) => path.extname ? path.extname.slice(1) : ''); // extension without dot
+            }
+        } else if (m[3]) {
+            switch (m[3]) {
+                case '%f':
+                    parts.push((path: Path) => path.name);
+                    break;
+                case '%n':
+                    parts.push((path: Path) => path.basename);
+                    break;
+                case '%p': // stringify the path by replacing / with _
+                    parts.push((path: Path) => {
+                        let p = path.value;
+                        if (p.startsWith('/')) {
+                            p = p.substring(1);
+                        }
+                        return p.replaceAll('/', '_');
+                    });
+                    break;
+                case '%i': // index
+                    parts.push((_path: Path, index: number) => String(index));
+                    break;
+                default: throw new Error(`Bug: should never happen`);
+            }
+        }
+        lastIndex = m.index + m[0].length;
+    }
+    if (!parts.length) {
+        return () => path;
+    } else {
+        if (lastIndex < path.length) {
+            const literal = path.substring(lastIndex);
+            parts.push(() => literal);
+        }
+        return (path: string, index: number) => {
+            const pathObj = new Path(truncPath(path));
+            const out = [];
+            for (const part of parts) {
+                out.push(part(pathObj, index));
+            }
+            return out.join('');
+        }
+    }
+}
+
+
+export class Path {
+    _name?: string;
+    _extname?: string;
+    _dirname?: string;
+    _basename?: string;
+
+    /**
+     * The complete path value
+     */
+    value: string
+
+    /**
+     * The file name (the last portion of the path). Includes the extension if present.
+     */
+    get name(): string {
+        if (!this._name) {
+            this._name = basename(this.value);
+        }
+        return this._name;
+    }
+
+    /**
+     * The extension of the file  including the leading '.'.
+     * An empty string if the file has no extension.
+     */
+    get extname(): string {
+        if (!this._extname) {
+            this._extname = extname(this.value);
+        }
+        return this._extname;
+    }
+    /**
+     * The directory portion of the path. Doesn'r include the trailing slash.
+     * If no directory is present, returns an empty string.
+     */
+    get dirname(): string {
+        if (!this._dirname) {
+            this._dirname = dirname(this.value);
+            if (this._dirname === '.') {
+                this._dirname = '';
+            }
+        }
+        return this._dirname;
+    }
+    /**
+     * The path without the extension
+     */
+    get basename(): string {
+        if (!this._basename) {
+            this._basename = this.extname ? this.name.slice(0, -this.extname.length) : this.name;
+        }
+        return this._basename;
+    }
+
+    constructor(value: string) {
+        this.value = value;
+    }
+
+    /**
+     * Return the complete path value (same as `value`)
+     */
+    toString(): string {
+        return this.value;
+    }
+}

package/src/utils/stream.test.ts

@@ -0,0 +1,13 @@
+import { describe, test, expect } from "vitest";
+import { BufferWritableStream } from "./stream";
+
+describe("BufferWritableStream", () => {
+    test("write buffer", () => {
+        const stream = new BufferWritableStream();
+        stream.write(Buffer.from("hello"));
+        stream.write(Buffer.from(" "));
+        stream.write(Buffer.from("world"));
+        stream.end();
+        expect(stream.getText()).toBe("hello world");
+    })
+})

package/src/utils/stream.ts

@@ -0,0 +1,27 @@
+import { Writable } from "stream";
+
+export class BufferWritableStream extends Writable {
+    chunks: Buffer[] = []
+    buffer: Buffer | undefined;
+
+    // _write method is required to handle the incoming data
+    _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+        this.chunks.push(chunk); // Collect the chunk into the array
+        callback(); // Indicate that the write is complete
+    }
+
+    // Optional _final method is called when the stream is ending
+    _final(callback: (error?: Error | null) => void) {
+        this.buffer = Buffer.concat(this.chunks); // Concatenate the collected chunks into a buffer
+        callback(); // Indicate the stream is finished
+    }
+
+    // Method to get the final buffer when the stream is closed
+    getBuffer() {
+        return this.buffer;
+    }
+
+    getText(encoding: BufferEncoding = "utf-8") {
+        return this.buffer?.toString(encoding);
+    }
+}

package/src/utils/tar.test.ts

@@ -0,0 +1,48 @@
+import { describe, test, expect, beforeAll, afterAll } from "vitest";
+import { loadTarIndex, TarBuilder, TarIndex } from "./tar";
+import { readFileSync, stat, statSync, unlinkSync } from "fs";
+
+const tarFile = `test-${Date.now()}.tar`;
+afterAll(() => {
+    unlinkSync(tarFile);
+});
+
+describe("Indexed tar format", () => {
+    const builder = new TarBuilder(tarFile);
+    test("build tar", async () => {
+        builder.add("file1.txt", Buffer.from("hello world!"));
+        builder.add("file2.txt", Buffer.from("bonjour monde!"));
+        builder.add("app/package.json", readFileSync("./package.json"));
+        await builder.build();
+        const stats = statSync(tarFile);
+        expect(stats.isFile()).toBeTruthy();
+    });
+
+    test("read tar", async () => {
+        const index = await loadTarIndex(tarFile) as TarIndex;
+        expect(index).toBeDefined();
+        expect(Object.keys(index.entries).length).toBe(3);
+        const file1 = index.get('file1.txt');
+        const file2 = index.get('file2.txt');
+        const file3 = index.get('app/package.json');
+        expect(file1).toBeDefined();
+        expect(file2).toBeDefined();
+        expect(file3).toBeDefined();
+        expect(file1!.size).toBe(12);
+        expect(file2!.size).toBe(14);
+        expect(file3!.size).toBeGreaterThan(0);
+
+        const content1 = await index.getContent('file1.txt');
+        const content2 = await index.getContent('file2.txt');
+        const content3 = await index.getContent('app/package.json');
+
+        expect(content1!.toString()).toBe("hello world!");
+        expect(content2!.toString()).toBe("bonjour monde!");
+        const pkg = JSON.parse(content3!.toString());
+        expect(pkg).toBeDefined();
+        expect(pkg).toHaveProperty("name");
+        expect(pkg).toHaveProperty("version");
+        expect(pkg).toHaveProperty("dependencies");
+    });
+
+});