@vertesia/memory 0.43.0

package/src/ContentObject.ts

@@ -0,0 +1,114 @@
+import { manyToMarkdown, pdfToText, pdfToTextBuffer, transformImage, transformImageToBuffer } from "@vertesia/converters";
+import fs from "fs";
+import { PassThrough, Readable } from "stream";
+import { Builder } from "./Builder.js";
+import { ContentSource } from "./ContentSource.js";
+
+export class ContentObject implements ContentSource {
+    constructor(public builder: Builder, public source: ContentSource, public encoding: BufferEncoding = "utf-8") { }
+
+    getContent(): Promise<Buffer> {
+        return this.source.getContent();
+    }
+
+    getStream(): Promise<NodeJS.ReadableStream> {
+        return this.source.getStream();
+    }
+
+    copyTo(entry: string) {
+        this.builder.addEntry(entry, this);
+    }
+
+    async copyToFile(file: string): Promise<fs.WriteStream> {
+
+        const input = await this.getStream();
+        const out = fs.createWriteStream(file);
+        const stream = input.pipe(out);
+        return new Promise((resolve, reject) => {
+            const handleError = (err: any) => {
+                reject(err);
+                out.close();
+            }
+            stream.on('finish', () => {
+                resolve(stream);
+                out.close();
+            });
+            input.on('error', handleError);
+            stream.on('error', handleError);
+        })
+    }
+
+    /**
+     * Get a text representation of the object
+     */
+    async getText(encoding?: BufferEncoding): Promise<string> {
+        const t = (await this.getContent()).toString(encoding || this.encoding);
+        return t;
+    }
+
+    getJsonValue(): Promise<any> {
+        return this.getText();
+    }
+}
+
+export class JsonObject extends ContentObject {
+    async getJsonValue(): Promise<any> {
+        return JSON.parse(await this.getText());
+    }
+}
+
+export interface MediaOptions {
+    max_hw?: number;
+    format?: "jpeg" | "png";
+}
+export class MediaObject extends ContentObject {
+    constructor(builder: Builder, source: ContentSource, public options: MediaOptions = {}) {
+        super(builder, source);
+        this.encoding = "base64";
+    }
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        const stream = await super.getStream();
+        const out = new PassThrough();
+        await transformImage(stream, out, this.options);
+        return out;
+    }
+    async getContent(): Promise<Buffer> {
+        const stream = await super.getStream();
+        return await transformImageToBuffer(stream, this.options);
+    }
+}
+
+
+export class PdfObject extends ContentObject {
+    constructor(builder: Builder, source: ContentSource) {
+        super(builder, source);
+    }
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        return Readable.from(await this.getContent());
+    }
+    async getContent(): Promise<Buffer> {
+        return pdfToTextBuffer(await super.getContent());
+    }
+    async getText(): Promise<string> {
+        return await pdfToText(await super.getContent());
+    }
+}
+
+
+export class DocxObject extends ContentObject {
+    constructor(builder: Builder, source: ContentSource) {
+        super(builder, source);
+    }
+    async getBuffer(): Promise<Buffer> {
+        return Buffer.from(await manyToMarkdown(await this.getStream(), "docx"), "utf-8");
+    }
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        const stream = new PassThrough();
+        stream.end(await this.getBuffer());
+        return stream;
+    }
+    async getText(): Promise<string> {
+        return await manyToMarkdown(await this.getStream(), "docx");
+    }
+
+}

package/src/ContentSource.ts

@@ -0,0 +1,88 @@
+import { createReadStream } from "fs";
+import { readFile } from "fs/promises";
+import { globSync } from 'glob';
+import { basename, extname, resolve } from "path";
+import { Readable } from "stream";
+
+export interface ContentSource {
+    getContent(): Promise<Buffer>;
+    getStream(): Promise<NodeJS.ReadableStream>;
+}
+
+export type SourceSpec = string | ContentSource;
+export abstract class AbstractContentSource implements ContentSource {
+    abstract getContent(): Promise<Buffer>
+
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        return Readable.from(await this.getContent());
+    }
+
+    static resolve(source: SourceSpec): ContentSource | ContentSource[] {
+        if (typeof source === 'string') {
+            return FileSource.resolve(source);
+        } else if (source instanceof AbstractContentSource) {
+            return source;
+        }
+        throw new Error("Unsupported content source: " + source);
+    }
+}
+
+export class BufferSource extends AbstractContentSource {
+    constructor(public buffer: Buffer) {
+        super()
+    }
+    getContent(): Promise<Buffer> {
+        return Promise.resolve(this.buffer);
+    }
+}
+
+export class TextSource extends AbstractContentSource {
+    constructor(public value: string, public encoding: BufferEncoding = "utf-8") {
+        super();
+    }
+
+    getContent() {
+        return Promise.resolve(Buffer.from(this.value, this.encoding));
+    }
+}
+
+export class FileSource extends AbstractContentSource {
+    file: string;
+    constructor(file: string, resolvePath = true) {
+        super();
+        this.file = resolvePath ? resolve(file) : file;
+    }
+
+    get path() {
+        return this.file;
+    }
+
+    get name() {
+        return basename(this.file);
+    }
+
+    get extname() {
+        return extname(this.file);
+    }
+
+    get nameWithoutExt() {
+        return this.name.slice(0, -this.extname.length);
+    }
+
+    getContent() {
+        return readFile(this.file);
+    }
+
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        return createReadStream(this.file);
+    }
+
+    static resolve(location: string): FileSource | FileSource[] {
+        if (location.includes('*')) {
+            return globSync(location, { absolute: true, withFileTypes: false }).map(f => new FileSource(f));
+        } else {
+            return new FileSource(location);
+        }
+    }
+
+}

package/src/MemoryPack.ts

@@ -0,0 +1,233 @@
+import micromatch from 'micromatch';
+import { extname } from "path";
+import { AbstractContentSource } from "./ContentSource.js";
+import { loadTarIndex, TarEntryIndex, TarIndex } from "./utils/tar.js";
+
+export const MEMORY_METADATA_ENTRY = "metadata.json";
+
+const EXPORT_CONTENT_KEY = '@content:';
+const EXPORT_ENTRY_KEY = '@file:';
+const EXPORT_PROPERTY_KEY = '@';
+const mediaExtensions = new Set([".jpg", ".jpeg", ".png"])
+
+/**
+ * Projection cannot contains both include and exclude keys
+ */
+export interface ProjectionProperties {
+    [key: string]: boolean | 0 | 1;
+}
+
+function applyProjection(projection: ProjectionProperties, object: any) {
+    const keys = Object.keys(projection);
+    if (keys.length < 1) {
+        return object;
+    }
+    const isInclusion = !!projection[keys[0]];
+    const out: any = {};
+    if (isInclusion) {
+        for (const key of Object.keys(object)) {
+            if (projection[key]) {
+                out[key] = object[key];
+            }
+        }
+    } else {
+        for (const key of Object.keys(object)) {
+            const value = projection[key];
+            if (value === undefined || value) {
+                out[key] = object[key];
+            }
+        }
+    }
+    return out;
+}
+
+
+export abstract class MemoryPack {
+    abstract readonly file: string;
+    abstract getMetadata(projection?: ProjectionProperties): Promise<Record<string, any>>;
+    abstract getEntry(path: string): MemoryEntry | null;
+    abstract getEntryContent(path: string): Promise<Buffer | null>;
+    abstract getEntryText(path: string, encoding?: BufferEncoding): Promise<string | null>;
+    abstract getPaths(filters?: string[]): string[];
+    abstract getEntries(filters?: string[]): MemoryEntry[];
+    abstract getEntriesContent(filters?: string[]): Promise<Buffer[]>;
+    abstract getEntriesText(filters?: string[], encoding?: BufferEncoding): Promise<string[]>;
+
+    async exportObject(mapping: Record<string, any>): Promise<Record<string, any>> {
+        let metadata: any;
+        const result: Record<string, any> = {};
+        for (const key of Object.keys(mapping)) {
+            const value = mapping[key];
+            if (typeof value === 'string') {
+                if (value === '@') {
+                    if (!metadata) {
+                        metadata = await this.getMetadata();
+                    }
+                    if (key === '@') {
+                        Object.assign(result, metadata);
+                    } else {
+                        result[key] = metadata;
+                    }
+                } else if (value.startsWith(EXPORT_CONTENT_KEY)) {
+                    const selector = value.slice(EXPORT_CONTENT_KEY.length);
+                    const isMulti = selector.includes('*') || selector.includes(',');
+                    if (isMulti) {
+                        result[key] = await this.getEntriesText(selector.split(','));
+                    } else {
+                        result[key] = await this.getEntryText(selector);
+                    }
+                } else if (value.startsWith(EXPORT_ENTRY_KEY)) {
+                    const selector = value.slice(EXPORT_ENTRY_KEY.length);
+                    const isMulti = selector.includes('*') || selector.includes(',');
+                    if (isMulti) {
+                        const entries = this.getEntries(selector.split(','));
+                        result[key] = await Promise.all(entries.map(entry => entry.getText().then(content => ({ name: entry.name, content }))));
+                    } else {
+                        const entry = this.getEntry(selector);
+                        result[key] = entry ? { name: entry.name, content: await entry.getText() } : null;
+                    }
+                } else if (value.startsWith(EXPORT_PROPERTY_KEY)) {
+                    if (!metadata) {
+                        metadata = await this.getMetadata();
+                    }
+                    const accessor = value.substring(EXPORT_PROPERTY_KEY.length);
+                    result[key] = resolveProperty(metadata, accessor);
+                } else {
+                    result[key] = value;
+                }
+            } else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+}
+
+
+export class MemoryEntry extends AbstractContentSource {
+    constructor(public index: TarIndex, public name: string, public offset: number, public size: number) {
+        super();
+    }
+    getContent(): Promise<Buffer> {
+        return this.index.getContentAt(this.offset, this.size);
+    }
+    getText(encoding: BufferEncoding = "utf-8"): Promise<string> {
+        return this.getContent().then((b) => b.toString(encoding));
+    }
+}
+
+export class TarMemoryPack extends MemoryPack {
+    constructor(public file: string, public index: TarIndex) {
+        super();
+        if (!index.get(MEMORY_METADATA_ENTRY)) {
+            throw new Error("Invalid memory tar file. Context entry not found");
+        }
+    }
+    async getMetadata(projection?: ProjectionProperties): Promise<Record<string, any>> {
+        const content = await this.index.getContent(MEMORY_METADATA_ENTRY);
+        if (content) {
+            let metadata = JSON.parse(content.toString('utf-8'));
+            if (projection) {
+                metadata = applyProjection(projection, metadata);
+            }
+            return metadata;
+        } else {
+            throw new Error("Invalid memory tar file. Context entry not found");
+        }
+    }
+
+    getPaths(filters?: string[]) {
+        let paths = this.index.getSortedPaths();
+        if (filters && filters.length > 0) {
+            paths = micromatch(paths, filters);
+        }
+        return paths;
+    }
+
+    getEntries(filters?: string[]) {
+        return this._getEntries(this.getPaths(filters));
+    }
+
+    private _getEntries(paths: string[]) {
+        const entries: MemoryEntry[] = [];
+        const index = this.index;
+        for (const path of paths) {
+            const entry: TarEntryIndex = index.get(path);
+            entries.push(new MemoryEntry(index, path, entry.offset, entry.size));
+        }
+        return entries;
+    }
+
+    getEntriesContent(filters?: string[]) {
+        const paths = this.getPaths(filters);
+        const promises: Promise<Buffer>[] = [];
+        for (const path of paths) {
+            promises.push(this.getEntryContent(path) as Promise<Buffer>);
+        }
+        return Promise.all(promises);
+    }
+
+    getEntriesText(filters?: string[], encoding?: BufferEncoding) {
+        const paths = this.getPaths(filters);
+        const promises: Promise<string>[] = [];
+        for (const path of paths) {
+            promises.push(this.getEntryText(path, encoding) as Promise<string>);
+        }
+        return Promise.all(promises);
+    }
+
+    getEntry(path: string) {
+        const entry = this.index.get(path);
+        return entry ? new MemoryEntry(this.index, path, entry.offset, entry.size) : null;
+    }
+
+    getEntryContent(path: string): Promise<Buffer | null> {
+        const entry = this.index.get(path);
+        if (!entry) {
+            return Promise.resolve(null);
+        } else {
+            return this.index.getContentAt(entry.offset, entry.size);
+        }
+    }
+
+    getEntryText(path: string, encoding?: BufferEncoding): Promise<string | null> {
+        return this.getEntryContent(path).then((b) => b?.toString(encoding || getTextEncodingForPath(path)) || null);
+    }
+
+    static async loadFile(file: string) {
+        const index = await loadTarIndex(file);
+        if (!index) {
+            throw new Error("Invalid memory tar file. Cannot load index");
+        }
+        return new TarMemoryPack(file, index);
+    }
+
+}
+
+
+export function loadMemoryPack(location: string): Promise<MemoryPack> {
+    // TODO we only support file paths as location for now
+    return TarMemoryPack.loadFile(location);
+}
+
+function getTextEncodingForPath(path: string) {
+    const ext = extname(path);
+    return mediaExtensions.has(ext) ? "base64" : "utf-8";
+}
+
+
+function resolveProperty(obj: Record<string, any>, key: string) {
+    if (key.includes('.')) {
+        const keys = key.split('.');
+        let value = obj;
+        for (const k of keys) {
+            value = value[k];
+            if (value == undefined) {
+                return undefined;
+            }
+        }
+        return value;
+    } else {
+        return obj[key];
+    }
+}

package/src/MemoryPackBuilder.ts

@@ -0,0 +1,55 @@
+import { Builder } from "./Builder.js";
+import { ContentSource } from "./ContentSource.js";
+import { MEMORY_METADATA_ENTRY, MemoryPack, ProjectionProperties } from "./MemoryPack.js";
+import { normalizePath, TarBuilder } from "./utils/tar.js";
+
+export interface FromOptions {
+    files?: string[];
+    projection?: ProjectionProperties;
+}
+
+export class MemoryPackBuilder {
+    baseMetadata: Record<string, any> = {};
+    entries: { [path: string]: ContentSource } = {};
+
+    constructor(public builder: Builder) {
+    }
+
+    async load(memory: MemoryPack, options: FromOptions = {}) {
+        const files = options.files || [];
+        // do not fetch the context entry an the metadata.json file
+        files.push(`!${MEMORY_METADATA_ENTRY}`);
+        const entries = memory.getEntries(files);
+        for (const entry of entries) {
+            this.add(entry.name, entry);
+        }
+        this.baseMetadata = await memory.getMetadata(options.projection);
+    }
+
+    add(path: string, content: ContentSource) {
+        path = normalizePath(path);
+        this.entries[path] = content;
+    }
+
+    async build(file: string, metadata: object) {
+        metadata = { ...this.baseMetadata, ...metadata };
+        if (!file.endsWith('.tar')) {
+            file += '.tar';
+        }
+        if (this.builder.options.gzip) {
+            file += ".gz";
+        }
+        const tar = new TarBuilder(file);
+        const keys = Object.keys(this.entries).sort();
+        for (const key of keys) {
+            const source = this.entries[key];
+            tar.add(key, await source.getContent());
+        }
+        tar.add(MEMORY_METADATA_ENTRY, Buffer.from(
+            JSON.stringify(metadata, undefined, this.builder.options.indent || undefined),
+            "utf-8")
+        );
+        await tar.build();
+        return file;
+    }
+}

package/src/builder.test.ts

@@ -0,0 +1,214 @@
+import { mkdtempSync, rmSync, statSync } from "fs";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { Builder } from "./Builder";
+import { loadTarIndex } from "./utils/tar";
+import { loadMemoryPack, MEMORY_METADATA_ENTRY } from "./MemoryPack";
+
+const memoryBaseFile = 'test-base-memory.tar';
+const memoryFile = 'test-memory.tar';
+const tmpdir = mkdtempSync("composable-memory-pack-test-");
+
+afterAll(() => {
+    rmSync(memoryBaseFile);
+    rmSync(memoryFile);
+    rmSync(tmpdir, { recursive: true });
+})
+
+describe("Builder", () => {
+    test("create base memory pack", async () => {
+        const builder = new Builder({
+            out: memoryBaseFile,
+        });
+
+        await builder.exec(`printf 'file1 from base memory' > ${tmpdir}/file1.txt`);
+        const file2 = await builder.exec('printf "file2 from base memory"', { quiet: true });
+        builder.copy(`${tmpdir}/file1.txt`, "file1.txt");
+        builder.copyText(file2 || '', "file2.txt");
+        builder.copyText("file3 from base memory", "file3.txt");
+
+        await builder.build({
+            baseProp1: "baseProp1",
+            baseProp2: "baseProp2",
+            baseProp3: "baseProp3",
+        });
+
+        const stats = statSync(memoryBaseFile);
+        expect(stats.isFile()).toBeTruthy();
+    });
+
+    test("create memory pack override", async () => {
+        const builder = new Builder({
+            out: memoryFile,
+        });
+
+        await builder.from(memoryBaseFile, {
+            files: ["!file1.txt"], // remove file1
+            projection: {
+                baseProp1: false, // remove baseProp1
+            }
+        })
+        // override file2
+        builder.copyText(`file2 from new memory`, "file2.txt");
+        builder.copyText(`file4 from new memory`, "file4.txt");
+
+        await builder.build({
+            baseProp2: "baseProp2", // override baseProp2
+            prop4: "prop4",
+            parent: { child: 123 }
+        });
+
+        const stats = statSync(memoryFile);
+        expect(stats.isFile()).toBeTruthy();
+
+        // check the content of the tar
+
+        const memory = await loadMemoryPack(memoryFile);
+        const entries = memory.getEntries();
+        expect(entries.length).toBe(4);
+        expect(entries.map(e => e.name).sort()).toStrictEqual(["file2.txt", "file3.txt", "file4.txt", MEMORY_METADATA_ENTRY].sort());
+
+        const entry1 = memory.getEntry("file1.txt");
+        expect(entry1).toBeNull();
+        const entry2 = memory.getEntry("file2.txt");
+        expect(entry2).not.toBeNull();
+        const entry3 = memory.getEntry("file3.txt");
+        expect(entry3).not.toBeNull();
+        const entry4 = memory.getEntry("file4.txt");
+        expect(entry4).not.toBeNull();
+
+        const content2 = (await entry2?.getContent())?.toString("utf-8");
+        const content3 = (await entry3?.getContent())?.toString("utf-8");
+        const content4 = (await entry4?.getContent())?.toString("utf-8");
+
+        expect(content2).toBe("file2 from new memory");
+        expect(content3).toBe("file3 from base memory");
+        expect(content4).toBe("file4 from new memory");
+
+        const context = await memory.getMetadata();
+        expect(context).toStrictEqual({
+            baseProp2: "baseProp2",
+            baseProp3: "baseProp3",
+            prop4: "prop4",
+            parent: { child: 123 }
+        });
+    })
+
+    test("MemoryPack.exportObject", async () => {
+        const memory = await loadMemoryPack(memoryFile);
+        let obj = await memory.exportObject({
+            "@": "@",
+            "file2": "@content:file2.txt",
+        });
+        expect(obj).toStrictEqual({
+            "file2": "file2 from new memory",
+            baseProp2: "baseProp2",
+            baseProp3: "baseProp3",
+            prop4: "prop4",
+            parent: { child: 123 }
+        })
+        obj = await memory.exportObject({
+            "prop4": "@prop4",
+            "file2": "@content:file2.txt",
+            "manifest": "@",
+        });
+        expect(obj).toStrictEqual({
+            prop4: "prop4",
+            "file2": "file2 from new memory",
+            manifest: {
+                baseProp2: "baseProp2",
+                baseProp3: "baseProp3",
+                prop4: "prop4",
+                parent: { child: 123 }
+            }
+        })
+        obj = await memory.exportObject({
+            "content": "@content:*.txt",
+        });
+        expect(obj).toStrictEqual({
+            content: [
+                "file2 from new memory",
+                "file3 from base memory",
+                "file4 from new memory"
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "content": "@content:file2.txt,file3.txt",
+        });
+        expect(obj).toStrictEqual({
+            content: [
+                "file2 from new memory",
+                "file3 from base memory",
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "content": "@content:file2.txt,*4.txt",
+        });
+        expect(obj).toStrictEqual({
+            content: [
+                "file2 from new memory",
+                "file4 from new memory"
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "files": "@file:*.txt",
+        });
+        expect(obj).toStrictEqual({
+            files: [
+                { name: "file2.txt", content: "file2 from new memory" },
+                { name: "file3.txt", content: "file3 from base memory" },
+                { name: "file4.txt", content: "file4 from new memory" }
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "files": "@file:file2.txt,file3.txt",
+        });
+        expect(obj).toStrictEqual({
+            files: [
+                { name: "file2.txt", content: "file2 from new memory" },
+                { name: "file3.txt", content: "file3 from base memory" },
+            ]
+        })
+
+        obj = await memory.exportObject({
+            "files": "@file:*2.txt,*4.txt",
+        });
+        expect(obj).toStrictEqual({
+            files: [
+                { name: "file2.txt", content: "file2 from new memory" },
+                { name: "file4.txt", content: "file4 from new memory" },
+            ]
+        })
+
+    })
+
+    test("MemoryPack.exportObject with nested props", async () => {
+        const memory = await loadMemoryPack(memoryFile);
+        let obj = await memory.exportObject({
+            "child": "@parent.child",
+        });
+        expect(obj).toStrictEqual({
+            child: 123
+        })
+    });
+
+    test("MemoryPack.exportObject with runtime values", async () => {
+        const memory = await loadMemoryPack(memoryFile);
+        let obj = await memory.exportObject({
+            "father": "@parent",
+            "instruction": "Use this runtime instruction",
+            someNullValue: null,
+            someObject: { msg: "Hello" }
+        });
+        expect(obj).toStrictEqual({
+            father: { child: 123 },
+            "instruction": "Use this runtime instruction",
+            someNullValue: null,
+            someObject: { msg: "Hello" }
+        })
+    });
+
+});