@vertesia/memory 0.24.0-dev.202601221707

package/src/MemoryPackBuilder.ts

@@ -0,0 +1,55 @@
+import { Builder } from "./Builder.js";
+import { ContentSource } from "./ContentSource.js";
+import { MEMORY_METADATA_ENTRY, MemoryPack, ProjectionProperties } from "./MemoryPack.js";
+import { normalizePath, TarBuilder } from "./utils/tar.js";
+
+export interface FromOptions {
+    files?: string[];
+    projection?: ProjectionProperties;
+}
+
+export class MemoryPackBuilder {
+    baseMetadata: Record<string, any> = {};
+    entries: { [path: string]: ContentSource } = {};
+
+    constructor(public builder: Builder) {
+    }
+
+    async load(memory: MemoryPack, options: FromOptions = {}) {
+        const files = options.files || [];
+        // do not fetch the context entry an the metadata.json file
+        files.push(`!${MEMORY_METADATA_ENTRY}`);
+        const entries = memory.getEntries(files);
+        for (const entry of entries) {
+            this.add(entry.name, entry);
+        }
+        this.baseMetadata = await memory.getMetadata(options.projection);
+    }
+
+    add(path: string, content: ContentSource) {
+        path = normalizePath(path);
+        this.entries[path] = content;
+    }
+
+    async build(file: string, metadata: object) {
+        metadata = { ...this.baseMetadata, ...metadata };
+        if (!file.endsWith('.tar')) {
+            file += '.tar';
+        }
+        if (this.builder.options.gzip) {
+            file += ".gz";
+        }
+        const tar = new TarBuilder(file);
+        const keys = Object.keys(this.entries).sort();
+        for (const key of keys) {
+            const source = this.entries[key];
+            tar.add(key, await source.getContent());
+        }
+        tar.add(MEMORY_METADATA_ENTRY, Buffer.from(
+            JSON.stringify(metadata, undefined, this.builder.options.indent || undefined),
+            "utf-8")
+        );
+        await tar.build();
+        return file;
+    }
+}

package/src/builder.test.ts

@@ -0,0 +1,214 @@
+import { mkdtempSync, rmSync, statSync } from "fs";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { Builder } from "./Builder";
+import { loadTarIndex } from "./utils/tar";
+import { loadMemoryPack, MEMORY_METADATA_ENTRY } from "./MemoryPack";
+
+const memoryBaseFile = 'test-base-memory.tar';
+const memoryFile = 'test-memory.tar';
+const tmpdir = mkdtempSync("composable-memory-pack-test-");
+
+afterAll(() => {
+    rmSync(memoryBaseFile);
+    rmSync(memoryFile);
+    rmSync(tmpdir, { recursive: true });
+})
+
+describe("Builder", () => {
+    test("create base memory pack", async () => {
+        const builder = new Builder({
+            out: memoryBaseFile,
+        });
+
+        await builder.exec(`printf 'file1 from base memory' > ${tmpdir}/file1.txt`);
+        const file2 = await builder.exec('printf "file2 from base memory"', { quiet: true });
+        builder.copy(`${tmpdir}/file1.txt`, "file1.txt");
+        builder.copyText(file2 || '', "file2.txt");
+        builder.copyText("file3 from base memory", "file3.txt");
+
+        await builder.build({
+            baseProp1: "baseProp1",
+            baseProp2: "baseProp2",
+            baseProp3: "baseProp3",
+        });
+
+        const stats = statSync(memoryBaseFile);
+        expect(stats.isFile()).toBeTruthy();
+    });
+
+    test("create memory pack override", async () => {
+        const builder = new Builder({
+            out: memoryFile,
+        });
+
+        await builder.from(memoryBaseFile, {
+            files: ["!file1.txt"], // remove file1
+            projection: {
+                baseProp1: false, // remove baseProp1
+            }
+        })
+        // override file2
+        builder.copyText(`file2 from new memory`, "file2.txt");
+        builder.copyText(`file4 from new memory`, "file4.txt");
+
+        await builder.build({
+            baseProp2: "baseProp2", // override baseProp2
+            prop4: "prop4",
+            parent: { child: 123 }
+        });
+
+        const stats = statSync(memoryFile);
+        expect(stats.isFile()).toBeTruthy();
+
+        // check the content of the tar
+
+        const memory = await loadMemoryPack(memoryFile);
+        const entries = memory.getEntries();
+        expect(entries.length).toBe(4);
+        expect(entries.map(e => e.name).sort()).toStrictEqual(["file2.txt", "file3.txt", "file4.txt", MEMORY_METADATA_ENTRY].sort());
+
+        const entry1 = memory.getEntry("file1.txt");
+        expect(entry1).toBeNull();
+        const entry2 = memory.getEntry("file2.txt");
+        expect(entry2).not.toBeNull();
+        const entry3 = memory.getEntry("file3.txt");
+        expect(entry3).not.toBeNull();
+        const entry4 = memory.getEntry("file4.txt");
+        expect(entry4).not.toBeNull();
+
+        const content2 = (await entry2?.getContent())?.toString("utf-8");
+        const content3 = (await entry3?.getContent())?.toString("utf-8");
+        const content4 = (await entry4?.getContent())?.toString("utf-8");
+
+        expect(content2).toBe("file2 from new memory");
+        expect(content3).toBe("file3 from base memory");
+        expect(content4).toBe("file4 from new memory");
+
+        const context = await memory.getMetadata();
+        expect(context).toStrictEqual({
+            baseProp2: "baseProp2",
+            baseProp3: "baseProp3",
+            prop4: "prop4",
+            parent: { child: 123 }
+        });
+    })
+
+    test("MemoryPack.exportObject", async () => {
+        const memory = await loadMemoryPack(memoryFile);
+        let obj = await memory.exportObject({
+            "@": "@",
+            "file2": "@content:file2.txt",
+        });
+        expect(obj).toStrictEqual({
+            "file2": "file2 from new memory",
+            baseProp2: "baseProp2",
+            baseProp3: "baseProp3",
+            prop4: "prop4",
+            parent: { child: 123 }
+        })
+        obj = await memory.exportObject({
+            "prop4": "@prop4",
+            "file2": "@content:file2.txt",
+            "manifest": "@",
+        });
+        expect(obj).toStrictEqual({
+            prop4: "prop4",
+            "file2": "file2 from new memory",
+            manifest: {
+                baseProp2: "baseProp2",
+                baseProp3: "baseProp3",
+                prop4: "prop4",
+                parent: { child: 123 }
+            }
+        })
+        obj = await memory.exportObject({
+            "content": "@content:*.txt",
+        });
+        expect(obj).toStrictEqual({
+            content: [
+                "file2 from new memory",
+                "file3 from base memory",
+                "file4 from new memory"
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "content": "@content:file2.txt,file3.txt",
+        });
+        expect(obj).toStrictEqual({
+            content: [
+                "file2 from new memory",
+                "file3 from base memory",
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "content": "@content:file2.txt,*4.txt",
+        });
+        expect(obj).toStrictEqual({
+            content: [
+                "file2 from new memory",
+                "file4 from new memory"
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "files": "@file:*.txt",
+        });
+        expect(obj).toStrictEqual({
+            files: [
+                { name: "file2.txt", content: "file2 from new memory" },
+                { name: "file3.txt", content: "file3 from base memory" },
+                { name: "file4.txt", content: "file4 from new memory" }
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "files": "@file:file2.txt,file3.txt",
+        });
+        expect(obj).toStrictEqual({
+            files: [
+                { name: "file2.txt", content: "file2 from new memory" },
+                { name: "file3.txt", content: "file3 from base memory" },
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "files": "@file:*2.txt,*4.txt",
+        });
+        expect(obj).toStrictEqual({
+            files: [
+                { name: "file2.txt", content: "file2 from new memory" },
+                { name: "file4.txt", content: "file4 from new memory" },
+            ]
+        })
+
+    })
+
+    test("MemoryPack.exportObject with nested props", async () => {
+        const memory = await loadMemoryPack(memoryFile);
+        let obj = await memory.exportObject({
+            "child": "@parent.child",
+        });
+        expect(obj).toStrictEqual({
+            child: 123
+        })
+    });
+
+    test("MemoryPack.exportObject with runtime values", async () => {
+        const memory = await loadMemoryPack(memoryFile);
+        let obj = await memory.exportObject({
+            "father": "@parent",
+            "instruction": "Use this runtime instruction",
+            someNullValue: null,
+            someObject: { msg: "Hello" }
+        });
+        expect(obj).toStrictEqual({
+            father: { child: 123 },
+            "instruction": "Use this runtime instruction",
+            someNullValue: null,
+            someObject: { msg: "Hello" }
+        })
+    });
+
+});

package/src/commands/copy.ts

@@ -0,0 +1,53 @@
+import { Builder } from "../Builder.js";
+import { DocxObject, MediaObject, MediaOptions, PdfObject } from "../ContentObject.js";
+import { AbstractContentSource, ContentSource, FileSource, SourceSpec } from "../ContentSource.js";
+import { createPathRewrite, PathMapperFn } from "../utils/rewrite.js";
+
+function rewritePath(source: ContentSource, index: number, mapper: PathMapperFn) {
+    const path = source instanceof FileSource ? source.file : '';
+    return mapper(path, index);
+}
+
+export interface CopyOptions {
+    media?: MediaOptions;
+    extractText?: boolean | string;
+}
+export function copy(builder: Builder, source: SourceSpec, toPath: string, options: CopyOptions = {}) {
+    const resolved = AbstractContentSource.resolve(source);
+    const mapperFn = createPathRewrite(toPath);
+    if (Array.isArray(resolved)) {
+        for (let i = 0, l = resolved.length; i < l; i++) {
+            const cs = resolved[i];
+            copyOne(builder, cs, rewritePath(cs, i, mapperFn), options);
+        }
+    } else {
+        return copyOne(builder, resolved, rewritePath(resolved, 0, mapperFn), options);
+    }
+}
+
+function copyOne(builder: Builder, source: ContentSource, toPath: string, options: CopyOptions) {
+    if (options.media) {
+        new MediaObject(builder, source, options.media).copyTo(toPath);
+    } else if (options.extractText) {
+        let type = options.extractText;
+        if (typeof type === "boolean") {
+            if (source instanceof FileSource) {
+                type = source.extname ? source.extname.slice(1) : source.extname;
+            } else {
+                throw new Error("source type for extractText must be specified");
+            }
+        }
+        switch (type) {
+            case "pdf":
+                new PdfObject(builder, source).copyTo(toPath);
+                break;
+            case "docx":
+                new DocxObject(builder, source).copyTo(toPath);
+                break;
+            default:
+                throw new Error("Unsupported extractText type: " + type);
+        }
+    } else {
+        builder.addEntry(toPath, source);
+    }
+}

package/src/commands/exec.test.ts

@@ -0,0 +1,22 @@
+import { describe, test, expect } from "vitest";
+import { exec } from "./exec";
+import fs from "fs";;
+
+describe("exec", () => {
+    test("test pipe", async () => {
+        const output = await exec('echo "hello" | wc -c', { quiet: true });
+        expect(output).toBeDefined();
+        expect(output!.trim()).toBe("6");
+    });
+    test("test pipe with redirection", async () => {
+        const name = "test-" + Date.now().toString() + ".txt";
+        const output = await exec(`echo "hello" | wc -c > ${name}`, { quiet: true });
+        try {
+            expect(output).toBeUndefined();
+            const content = fs.readFileSync(name, 'utf-8').trim();
+            expect(content).toBe("6");
+        } finally {
+            fs.unlinkSync(name);
+        }
+    });
+});

package/src/commands/exec.ts

@@ -0,0 +1,83 @@
+import { ChildProcess, spawn } from 'child_process';
+import fs from 'fs';
+import { Stream, Writable } from 'stream';
+import { Command, splitPipeCommands } from '../utils/cmdline.js';
+import { BufferWritableStream } from '../utils/stream.js';
+
+export interface ExecOptions {
+    quiet?: boolean;
+}
+
+export async function exec(commandLine: string, options: ExecOptions = {}): Promise<string | undefined> {
+    const verbose = !options.quiet;
+    commandLine = commandLine.trim();
+    const { commands, out } = splitPipeCommands(commandLine);
+    if (!commands.length) {
+        throw new Error('Invalid command line. No command: ' + commandLine);
+    }
+
+    let outStream: Writable;
+    if (out) {
+        outStream = fs.createWriteStream(out, { flags: 'w' });
+    } else {
+        outStream = new BufferWritableStream();
+    }
+    const pipePromise = executePipe(commands, outStream, verbose);
+    const outPromise = new Promise((resolve, reject) => {
+        outStream.on('finish', () => {
+            resolve(undefined);
+        });
+        outStream.on('error', (err: Error) => {
+            reject(err);
+            outStream.destroy();
+        });
+    });
+
+    const [status] = await Promise.all([pipePromise, outPromise]);
+
+    if (verbose) {
+        if (!status) {
+            console.log(`Command: ${commandLine} exited with status ${status}`);
+        } else {
+            console.error(`Command: ${commandLine} exited with status ${status}`);
+        }
+    }
+
+    if (outStream instanceof BufferWritableStream) {
+        return outStream.getText();
+    } else {
+        return undefined;
+    }
+}
+
+
+export function executePipe(commands: Command[], finalOutput: Writable | undefined, verbose: boolean = false) {
+    return new Promise((resolve, reject) => {
+        let input: Stream | undefined;
+        let child: ChildProcess | undefined;
+        for (const cmd of commands) {
+            verbose && console.log(`Running: ${cmd.name} ${cmd.args?.join(' ')}`);
+            child = spawn(cmd.name, cmd.args, {
+                // in, out, err
+                stdio: ['pipe', 'pipe', 'inherit'],
+            });
+            if (input) {
+                input.pipe(child.stdin!);
+            }
+            input = child.stdout!;
+            child.on("error", (err: Error) => {
+                console.error(`Failed to run ${cmd.name}`, err);
+                reject(err);
+            })
+        }
+        if (child) {
+            child.on("exit", (code: number | null, signal: string | null) => {
+                resolve(code !== null ? code : signal);
+            });
+            finalOutput && child.stdout?.pipe(finalOutput);
+        } else {
+            reject(new Error("no child spawned"));
+        }
+    });
+
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export { Builder, buildMemoryPack } from "./Builder.js"
+export type { BuildOptions, Commands } from "./Builder.js"
+export type { CopyOptions } from "./commands/copy.js"
+export type { ExecOptions } from "./commands/exec.js"
+export { ContentObject, DocxObject, JsonObject, MediaObject, PdfObject } from "./ContentObject.js"
+export type { MediaOptions } from "./ContentObject.js"
+export { AbstractContentSource, BufferSource, FileSource, TextSource } from "./ContentSource.js"
+export type { ContentSource, SourceSpec } from "./ContentSource.js"
+export { MemoryEntry, TarMemoryPack, loadMemoryPack } from "./MemoryPack.js"
+export type { MemoryPack, ProjectionProperties } from "./MemoryPack.js"
+export { MemoryPackBuilder } from "./MemoryPackBuilder.js"
+export type { FromOptions } from "./MemoryPackBuilder.js"
+export { TarBuilder, TarIndex, loadTarIndex } from './utils/tar.js'
+export type { TarEntry, TarEntryIndex } from './utils/tar.js'

package/src/utils/cmdline.test.ts

@@ -0,0 +1,32 @@
+import { describe, test, expect } from "vitest";
+import { splitCommandLine, splitPipeCommands } from "./cmdline";
+
+describe("command line parser", () => {
+
+    test("split command line", () => {
+        const args = splitCommandLine("cmd -m \"hello 'world'\" 'some \"file\"' 'nested \"dquote\"' \"nested \'squote\'\"");
+        expect(args).toEqual(["cmd", "-m", "hello 'world'", "some \"file\"", 'nested "dquote"', "nested 'squote'"]);
+
+    })
+
+    test("split commands pipeline", () => {
+        const pipe = splitPipeCommands("cat \"the file.txt\" | grep \"the\" | wc -c");
+        expect(pipe.out).toBeUndefined();
+        expect(pipe.commands).toEqual([
+            { name: "cat", args: ["the file.txt"] },
+            { name: "grep", args: ["the"] },
+            { name: "wc", args: ["-c"] }
+        ]);
+    })
+
+    test("split commands pipeline and redirect output", () => {
+        const pipe = splitPipeCommands("cat \"the file.txt\" | grep \"the\" | wc -c > out.txt");
+        expect(pipe.out).toBe("out.txt");
+        expect(pipe.commands).toEqual([
+            { name: "cat", args: ["the file.txt"] },
+            { name: "grep", args: ["the"] },
+            { name: "wc", args: ["-c"] }
+        ]);
+    })
+
+});

package/src/utils/cmdline.ts

@@ -0,0 +1,92 @@
+
+function readWord(text: string, index: number): [string, number] {
+    let i = index, len = text.length;
+    while (i < len && (text[i] !== ' ' || text[i] !== '\t')) {
+        const c = text[i];
+        if (c === ' ' || c === '\t') {
+            return [text.substring(index, i), i];
+        }
+        i++;
+    }
+    return [text.substring(index), len];
+}
+
+function readQuotedArg(text: string, index: number, quote: string): [string, number] {
+    let i = index, len = text.length;
+    while (i < len) {
+        const c = text[i];
+        if (c === '\\') {
+            i += 2;
+            continue;
+        } else if (c === quote) {
+            return [text.substring(index, i), i + 1];
+        } else {
+            i++;
+        }
+    }
+    return [text.substring(index), len];
+}
+
+export function splitCommandLine(text: string) {
+    const args = [];
+    let i = 0, len = text.length;
+    while (i < len) {
+        let c = text[i];
+        if (c === ' ' || c === '\t') {
+            i++; // skip whitespace
+        } else if (c === '"' || c === "'") {
+            const [word, offset] = readQuotedArg(text, i + 1, c);
+            i = offset;
+            args.push(word);
+        } else {
+            const [word, offset] = readWord(text, i);
+            i = offset;
+            args.push(word);
+        }
+    }
+    return args;
+}
+
+export interface Command {
+    name: string;
+    args: string[];
+}
+export interface CommandPipe {
+    commands: Command[];
+    out?: string | undefined;
+}
+export function splitPipeCommands(text: string): CommandPipe {
+    const tokens = splitCommandLine(text);
+    const commands: Command[] = [];
+    let args: string[] = [];
+    for (const token of tokens) {
+        if (token === "|") {
+            if (args.length < 1) {
+                throw new Error("Invalid pipe character. Expecting a command first.");
+            }
+            const name = args.shift()!;
+            commands.push({
+                name,
+                args
+            });
+            args = [];
+        } else {
+            args.push(token);
+        }
+    }
+    let out: string | undefined;
+    if (args.length > 0) {
+        const name = args.shift()!;
+        if (args.length > 1 && args[args.length - 2] === ">") {
+            out = args.pop();
+            args.pop(); // remove the ">"
+        }
+        commands.push({
+            name,
+            args
+        });
+    }
+    return {
+        commands, out
+    };
+}

package/src/utils/rewrite.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, test } from "vitest";
+import { createPathRewrite } from "./rewrite";
+
+describe("Path rewrite", () => {
+    test("constant path", () => {
+        let fn = createPathRewrite("/root!some/path")
+        let r = fn("/root/the/path", 0)
+        expect(r).toBe("some/path")
+
+        fn = createPathRewrite("some/path")
+        r = fn("/root/the/path", 0)
+        expect(r).toBe("some/path")
+    })
+
+    test("wildcard path", () => {
+        let fn = createPathRewrite("/root!*")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("the/path.ext")
+
+        fn = createPathRewrite("*")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("path.ext")
+    })
+
+    test("expression path", () => {
+        let fn = createPathRewrite("/root!images/%d/file-%n-%i.%e")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the/file-path-0.ext")
+
+        fn = createPathRewrite("images/%d/file-%n-%i.%e")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/file-path-0.ext")
+    })
+
+    test("expression path using %f", () => {
+        let fn = createPathRewrite("/root!images/%d/file-%i-%f")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the/file-0-path.ext")
+
+        fn = createPathRewrite("images/%d/file-%i-%f")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/file-0-path.ext")
+    })
+
+    test("expression using wildcard with prefix", () => {
+        let fn = createPathRewrite("/root!images/*")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the/path.ext")
+
+        fn = createPathRewrite("images/*")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/path.ext")
+    })
+
+    test("expression using %p", () => {
+        let fn = createPathRewrite("/root!images/%p")
+        let r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/the_path.ext")
+
+        fn = createPathRewrite("images/%p")
+        r = fn("/root/the/path.ext", 0)
+        expect(r).toBe("images/path.ext")
+    })
+
+})