@vertesia/memory 0.24.0-dev.202601221707

package/src/Builder.ts

@@ -0,0 +1,239 @@
+import { AsyncObjectWalker } from "@vertesia/json";
+import { mkdtempSync, rmSync } from "fs";
+import os from "os";
+import { join, resolve } from "path";
+import { copy, CopyOptions } from "./commands/copy.js";
+import { exec, ExecOptions } from "./commands/exec.js";
+import { ContentObject, DocxObject, JsonObject, MediaObject, MediaOptions, PdfObject } from "./ContentObject.js";
+import { AbstractContentSource, ContentSource, SourceSpec, TextSource } from "./ContentSource.js";
+import { loadMemoryPack } from "./MemoryPack.js";
+import { FromOptions, MemoryPackBuilder } from "./MemoryPackBuilder.js";
+
+export interface BuildOptions {
+    indent?: number;
+    /**
+     * the path to save the output. Defaults to 'memory.tar'.
+     * If no .tar extension is present it will be added
+     */
+    out?: string;
+
+    /**
+     * If set, suppress logs. Defaults to false.
+     */
+    quiet?: boolean;
+
+    /**
+     * If true, compress the output (tar or json) with gzip. Defaults to false.
+     */
+    gzip?: boolean;
+
+    /**
+     * Vars to be injected into the script context as the vars object
+     */
+    vars?: Record<string, any>;
+
+    /**
+     * Optional publish action
+     * @param file
+     * @returns the URI of the published memory
+     */
+    publish?: (file: string, name: string) => Promise<string>
+
+    /**
+     * The directory where to transpile the recipe ts file
+     */
+    transpileDir?: string;
+}
+
+export interface Commands {
+    vars: () => Record<string, any>;
+    tmpdir: () => string;
+    exec: (cmd: string, options?: ExecOptions) => Promise<void | string>;
+    from: (location: string, options?: FromOptions) => Promise<void>;
+    content: (location: string, encoding?: BufferEncoding) => ContentObject | ContentObject[];
+    json: (location: string) => ContentObject | ContentObject[];
+    pdf: (location: string) => PdfObject | PdfObject[];
+    docx: (location: string) => DocxObject | DocxObject[];
+    media: (location: string, options?: MediaOptions) => MediaObject | MediaObject[];
+    copy: (location: SourceSpec, path: string, options?: CopyOptions) => void;
+    copyText: (text: string, path: string) => void;
+}
+
+export class Builder implements Commands {
+    static instance?: Builder;
+
+    static getInstance() {
+        if (!Builder.instance) {
+            throw new Error("No builder instance found");
+        }
+        return Builder.instance;
+    }
+
+    _vars: Record<string, any>;
+    _tmpdir?: string;
+    memory: MemoryPackBuilder;
+
+    constructor(public options: BuildOptions = {}) {
+        this.memory = new MemoryPackBuilder(this);
+        this._vars = options.vars || {};
+        Builder.instance = this;
+    }
+
+    vars(): Record<string, any> {
+        return this._vars;
+    }
+
+    tmpdir() {
+        if (!this._tmpdir) {
+            this._tmpdir = mkdtempSync(join(os.tmpdir(), 'becomposable-memo-'));
+        }
+        return this._tmpdir;
+    }
+
+    from(location: string, options?: FromOptions) {
+        return loadMemoryPack(location).then((memory) => {
+            return this.memory.load(memory, options);
+        });
+    }
+
+    addEntry(path: string, source: ContentSource) {
+        this.memory.add(path, source);
+    }
+
+    copy(location: SourceSpec, path: string, options: CopyOptions = {}) {
+        copy(this, location, path, options);
+    }
+
+    copyText(text: string, path: string) {
+        copy(this, new TextSource(text), path);
+    }
+
+    exec(cmd: string, options?: ExecOptions) {
+        return exec(cmd, options);
+    }
+
+    content(location: string, encoding?: BufferEncoding) {
+        const source = AbstractContentSource.resolve(location);
+        if (Array.isArray(source)) {
+            return source.map(s => new ContentObject(this, s, encoding));
+        } else {
+            return new ContentObject(this, source, encoding);
+        }
+    }
+
+    json(location: string) {
+        const source = AbstractContentSource.resolve(location);
+        if (Array.isArray(source)) {
+            return source.map(s => new JsonObject(this, s));
+        } else {
+            return new JsonObject(this, source);
+        }
+    }
+
+    pdf(location: string) {
+        const source = AbstractContentSource.resolve(location);
+        if (Array.isArray(source)) {
+            return source.map(s => new PdfObject(this, s));
+        } else {
+            return new PdfObject(this, source);
+        }
+    }
+
+    docx(location: string) {
+        const source = AbstractContentSource.resolve(location);
+        if (Array.isArray(source)) {
+            return source.map(s => new DocxObject(this, s));
+        } else {
+            return new DocxObject(this, source);
+        }
+    }
+
+    media(location: string, options?: MediaOptions) {
+        const source = AbstractContentSource.resolve(location);
+        if (Array.isArray(source)) {
+            return source.map(s => new MediaObject(this, s, options));
+        } else {
+            return new MediaObject(this, source, options);
+        }
+    }
+
+    async build(object: Record<string, any>) {
+        try {
+            let { fileName, publishName } = this._getOutputNames();
+            // resolve all content objects values from the context object
+            object = await resolveContextObject(object);
+            // write the memory to a file
+            fileName = await this.memory.build(fileName, object);
+            let target: string = fileName;
+            if (publishName) {
+                try {
+                    target = await this.options.publish!(fileName, publishName);
+                } finally {
+                    rmSync(fileName);
+                }
+            }
+            this.options.quiet || console.log(`Memory saved to ${target}`);
+            return target;
+        } finally {
+            if (this._tmpdir) {
+                rmSync(this._tmpdir, { recursive: true });
+            }
+        }
+    }
+
+    private _getOutputNames(): { fileName: string, publishName: string | undefined } {
+        const options = this.options;
+        if (!options.out) {
+            options.out = "memory.tar";
+        }
+        let fileName: string;
+        let publishName: string | undefined;
+        const out = options.out;
+        if (out.startsWith("memory:")) {
+            if (!options.publish) {
+                throw new Error(`The publish option is required for "${out}" output`);
+            }
+            // force gzip when publishing
+            options.gzip = true;
+            // create a temporary path for the output
+            fileName = createTmpBaseName(this.tmpdir());
+            publishName = out.substring("memory:".length);
+        } else {
+            fileName = resolve(out);
+        }
+        return { fileName, publishName };
+    }
+
+}
+
+function resolveContextObject(object: Record<string, any>): Promise<Record<string, any>> {
+    return new AsyncObjectWalker().map(object, async (_key, value) => {
+        if (value instanceof ContentObject) {
+            return await value.getText();
+        } else {
+            return value;
+        }
+    });
+}
+
+function createTmpBaseName(tmpdir: string) {
+    return join(tmpdir, `.composable-memory-${Date.now()}.tar`);
+}
+
+
+export function buildMemoryPack(recipeFn: (commands: Commands) => Promise<Record<string, any>>, options: BuildOptions): Promise<string> {
+    const builder = new Builder(options);
+    return recipeFn({
+        vars: builder.vars.bind(builder),
+        tmpdir: builder.tmpdir.bind(builder),
+        exec: builder.exec.bind(builder),
+        from: builder.from.bind(builder),
+        content: builder.content.bind(builder),
+        json: builder.json.bind(builder),
+        pdf: builder.pdf.bind(builder),
+        docx: builder.docx.bind(builder),
+        media: builder.media.bind(builder),
+        copy: builder.copy.bind(builder),
+        copyText: builder.copyText.bind(builder),
+    }).then(metadata => builder.build(metadata));
+}

package/src/ContentObject.ts

@@ -0,0 +1,114 @@
+import { manyToMarkdown, pdfToText, pdfToTextBuffer, transformImage, transformImageToBuffer } from "@vertesia/converters";
+import fs from "fs";
+import { PassThrough, Readable } from "stream";
+import { Builder } from "./Builder.js";
+import { ContentSource } from "./ContentSource.js";
+
+export class ContentObject implements ContentSource {
+    constructor(public builder: Builder, public source: ContentSource, public encoding: BufferEncoding = "utf-8") { }
+
+    getContent(): Promise<Buffer> {
+        return this.source.getContent();
+    }
+
+    getStream(): Promise<NodeJS.ReadableStream> {
+        return this.source.getStream();
+    }
+
+    copyTo(entry: string) {
+        this.builder.addEntry(entry, this);
+    }
+
+    async copyToFile(file: string): Promise<fs.WriteStream> {
+
+        const input = await this.getStream();
+        const out = fs.createWriteStream(file);
+        const stream = input.pipe(out);
+        return new Promise((resolve, reject) => {
+            const handleError = (err: any) => {
+                reject(err);
+                out.close();
+            }
+            stream.on('finish', () => {
+                resolve(stream);
+                out.close();
+            });
+            input.on('error', handleError);
+            stream.on('error', handleError);
+        })
+    }
+
+    /**
+     * Get a text representation of the object
+     */
+    async getText(encoding?: BufferEncoding): Promise<string> {
+        const t = (await this.getContent()).toString(encoding || this.encoding);
+        return t;
+    }
+
+    getJsonValue(): Promise<any> {
+        return this.getText();
+    }
+}
+
+export class JsonObject extends ContentObject {
+    async getJsonValue(): Promise<any> {
+        return JSON.parse(await this.getText());
+    }
+}
+
+export interface MediaOptions {
+    max_hw?: number;
+    format?: "jpeg" | "png";
+}
+export class MediaObject extends ContentObject {
+    constructor(builder: Builder, source: ContentSource, public options: MediaOptions = {}) {
+        super(builder, source);
+        this.encoding = "base64";
+    }
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        const stream = await super.getStream();
+        const out = new PassThrough();
+        await transformImage(stream, out, this.options);
+        return out;
+    }
+    async getContent(): Promise<Buffer> {
+        const stream = await super.getStream();
+        return await transformImageToBuffer(stream, this.options);
+    }
+}
+
+
+export class PdfObject extends ContentObject {
+    constructor(builder: Builder, source: ContentSource) {
+        super(builder, source);
+    }
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        return Readable.from(await this.getContent());
+    }
+    async getContent(): Promise<Buffer> {
+        return pdfToTextBuffer(await super.getContent());
+    }
+    async getText(): Promise<string> {
+        return await pdfToText(await super.getContent());
+    }
+}
+
+
+export class DocxObject extends ContentObject {
+    constructor(builder: Builder, source: ContentSource) {
+        super(builder, source);
+    }
+    async getBuffer(): Promise<Buffer> {
+        return Buffer.from(await manyToMarkdown(await this.getStream(), "docx"), "utf-8");
+    }
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        const stream = new PassThrough();
+        stream.end(await this.getBuffer());
+        return stream;
+    }
+    async getText(): Promise<string> {
+        return await manyToMarkdown(await this.getStream(), "docx");
+    }
+
+}

package/src/ContentSource.ts

@@ -0,0 +1,88 @@
+import { createReadStream } from "fs";
+import { readFile } from "fs/promises";
+import { globSync } from 'glob';
+import { basename, extname, resolve } from "path";
+import { Readable } from "stream";
+
+export interface ContentSource {
+    getContent(): Promise<Buffer>;
+    getStream(): Promise<NodeJS.ReadableStream>;
+}
+
+export type SourceSpec = string | ContentSource;
+export abstract class AbstractContentSource implements ContentSource {
+    abstract getContent(): Promise<Buffer>
+
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        return Readable.from(await this.getContent());
+    }
+
+    static resolve(source: SourceSpec): ContentSource | ContentSource[] {
+        if (typeof source === 'string') {
+            return FileSource.resolve(source);
+        } else if (source instanceof AbstractContentSource) {
+            return source;
+        }
+        throw new Error("Unsupported content source: " + source);
+    }
+}
+
+export class BufferSource extends AbstractContentSource {
+    constructor(public buffer: Buffer) {
+        super()
+    }
+    getContent(): Promise<Buffer> {
+        return Promise.resolve(this.buffer);
+    }
+}
+
+export class TextSource extends AbstractContentSource {
+    constructor(public value: string, public encoding: BufferEncoding = "utf-8") {
+        super();
+    }
+
+    getContent() {
+        return Promise.resolve(Buffer.from(this.value, this.encoding));
+    }
+}
+
+export class FileSource extends AbstractContentSource {
+    file: string;
+    constructor(file: string, resolvePath = true) {
+        super();
+        this.file = resolvePath ? resolve(file) : file;
+    }
+
+    get path() {
+        return this.file;
+    }
+
+    get name() {
+        return basename(this.file);
+    }
+
+    get extname() {
+        return extname(this.file);
+    }
+
+    get nameWithoutExt() {
+        return this.name.slice(0, -this.extname.length);
+    }
+
+    getContent() {
+        return readFile(this.file);
+    }
+
+    async getStream(): Promise<NodeJS.ReadableStream> {
+        return createReadStream(this.file);
+    }
+
+    static resolve(location: string): FileSource | FileSource[] {
+        if (location.includes('*')) {
+            return globSync(location, { absolute: true, withFileTypes: false }).map(f => new FileSource(f));
+        } else {
+            return new FileSource(location);
+        }
+    }
+
+}

package/src/MemoryPack.ts

@@ -0,0 +1,233 @@
+import micromatch from 'micromatch';
+import { extname } from "path";
+import { AbstractContentSource } from "./ContentSource.js";
+import { loadTarIndex, TarEntryIndex, TarIndex } from "./utils/tar.js";
+
+export const MEMORY_METADATA_ENTRY = "metadata.json";
+
+const EXPORT_CONTENT_KEY = '@content:';
+const EXPORT_ENTRY_KEY = '@file:';
+const EXPORT_PROPERTY_KEY = '@';
+const mediaExtensions = new Set([".jpg", ".jpeg", ".png"])
+
+/**
+ * Projection cannot contains both include and exclude keys
+ */
+export interface ProjectionProperties {
+    [key: string]: boolean | 0 | 1;
+}
+
+function applyProjection(projection: ProjectionProperties, object: any) {
+    const keys = Object.keys(projection);
+    if (keys.length < 1) {
+        return object;
+    }
+    const isInclusion = !!projection[keys[0]];
+    const out: any = {};
+    if (isInclusion) {
+        for (const key of Object.keys(object)) {
+            if (projection[key]) {
+                out[key] = object[key];
+            }
+        }
+    } else {
+        for (const key of Object.keys(object)) {
+            const value = projection[key];
+            if (value === undefined || value) {
+                out[key] = object[key];
+            }
+        }
+    }
+    return out;
+}
+
+
+export abstract class MemoryPack {
+    abstract readonly file: string;
+    abstract getMetadata(projection?: ProjectionProperties): Promise<Record<string, any>>;
+    abstract getEntry(path: string): MemoryEntry | null;
+    abstract getEntryContent(path: string): Promise<Buffer | null>;
+    abstract getEntryText(path: string, encoding?: BufferEncoding): Promise<string | null>;
+    abstract getPaths(filters?: string[]): string[];
+    abstract getEntries(filters?: string[]): MemoryEntry[];
+    abstract getEntriesContent(filters?: string[]): Promise<Buffer[]>;
+    abstract getEntriesText(filters?: string[], encoding?: BufferEncoding): Promise<string[]>;
+
+    async exportObject(mapping: Record<string, any>): Promise<Record<string, any>> {
+        let metadata: any;
+        const result: Record<string, any> = {};
+        for (const key of Object.keys(mapping)) {
+            const value = mapping[key];
+            if (typeof value === 'string') {
+                if (value === '@') {
+                    if (!metadata) {
+                        metadata = await this.getMetadata();
+                    }
+                    if (key === '@') {
+                        Object.assign(result, metadata);
+                    } else {
+                        result[key] = metadata;
+                    }
+                } else if (value.startsWith(EXPORT_CONTENT_KEY)) {
+                    const selector = value.slice(EXPORT_CONTENT_KEY.length);
+                    const isMulti = selector.includes('*') || selector.includes(',');
+                    if (isMulti) {
+                        result[key] = await this.getEntriesText(selector.split(','));
+                    } else {
+                        result[key] = await this.getEntryText(selector);
+                    }
+                } else if (value.startsWith(EXPORT_ENTRY_KEY)) {
+                    const selector = value.slice(EXPORT_ENTRY_KEY.length);
+                    const isMulti = selector.includes('*') || selector.includes(',');
+                    if (isMulti) {
+                        const entries = this.getEntries(selector.split(','));
+                        result[key] = await Promise.all(entries.map(entry => entry.getText().then(content => ({ name: entry.name, content }))));
+                    } else {
+                        const entry = this.getEntry(selector);
+                        result[key] = entry ? { name: entry.name, content: await entry.getText() } : null;
+                    }
+                } else if (value.startsWith(EXPORT_PROPERTY_KEY)) {
+                    if (!metadata) {
+                        metadata = await this.getMetadata();
+                    }
+                    const accessor = value.substring(EXPORT_PROPERTY_KEY.length);
+                    result[key] = resolveProperty(metadata, accessor);
+                } else {
+                    result[key] = value;
+                }
+            } else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+}
+
+
+export class MemoryEntry extends AbstractContentSource {
+    constructor(public index: TarIndex, public name: string, public offset: number, public size: number) {
+        super();
+    }
+    getContent(): Promise<Buffer> {
+        return this.index.getContentAt(this.offset, this.size);
+    }
+    getText(encoding: BufferEncoding = "utf-8"): Promise<string> {
+        return this.getContent().then((b) => b.toString(encoding));
+    }
+}
+
+export class TarMemoryPack extends MemoryPack {
+    constructor(public file: string, public index: TarIndex) {
+        super();
+        if (!index.get(MEMORY_METADATA_ENTRY)) {
+            throw new Error("Invalid memory tar file. Context entry not found");
+        }
+    }
+    async getMetadata(projection?: ProjectionProperties): Promise<Record<string, any>> {
+        const content = await this.index.getContent(MEMORY_METADATA_ENTRY);
+        if (content) {
+            let metadata = JSON.parse(content.toString('utf-8'));
+            if (projection) {
+                metadata = applyProjection(projection, metadata);
+            }
+            return metadata;
+        } else {
+            throw new Error("Invalid memory tar file. Context entry not found");
+        }
+    }
+
+    getPaths(filters?: string[]) {
+        let paths = this.index.getSortedPaths();
+        if (filters && filters.length > 0) {
+            paths = micromatch(paths, filters);
+        }
+        return paths;
+    }
+
+    getEntries(filters?: string[]) {
+        return this._getEntries(this.getPaths(filters));
+    }
+
+    private _getEntries(paths: string[]) {
+        const entries: MemoryEntry[] = [];
+        const index = this.index;
+        for (const path of paths) {
+            const entry: TarEntryIndex = index.get(path);
+            entries.push(new MemoryEntry(index, path, entry.offset, entry.size));
+        }
+        return entries;
+    }
+
+    getEntriesContent(filters?: string[]) {
+        const paths = this.getPaths(filters);
+        const promises: Promise<Buffer>[] = [];
+        for (const path of paths) {
+            promises.push(this.getEntryContent(path) as Promise<Buffer>);
+        }
+        return Promise.all(promises);
+    }
+
+    getEntriesText(filters?: string[], encoding?: BufferEncoding) {
+        const paths = this.getPaths(filters);
+        const promises: Promise<string>[] = [];
+        for (const path of paths) {
+            promises.push(this.getEntryText(path, encoding) as Promise<string>);
+        }
+        return Promise.all(promises);
+    }
+
+    getEntry(path: string) {
+        const entry = this.index.get(path);
+        return entry ? new MemoryEntry(this.index, path, entry.offset, entry.size) : null;
+    }
+
+    getEntryContent(path: string): Promise<Buffer | null> {
+        const entry = this.index.get(path);
+        if (!entry) {
+            return Promise.resolve(null);
+        } else {
+            return this.index.getContentAt(entry.offset, entry.size);
+        }
+    }
+
+    getEntryText(path: string, encoding?: BufferEncoding): Promise<string | null> {
+        return this.getEntryContent(path).then((b) => b?.toString(encoding || getTextEncodingForPath(path)) || null);
+    }
+
+    static async loadFile(file: string) {
+        const index = await loadTarIndex(file);
+        if (!index) {
+            throw new Error("Invalid memory tar file. Cannot load index");
+        }
+        return new TarMemoryPack(file, index);
+    }
+
+}
+
+
+export function loadMemoryPack(location: string): Promise<MemoryPack> {
+    // TODO we only support file paths as location for now
+    return TarMemoryPack.loadFile(location);
+}
+
+function getTextEncodingForPath(path: string) {
+    const ext = extname(path);
+    return mediaExtensions.has(ext) ? "base64" : "utf-8";
+}
+
+
+function resolveProperty(obj: Record<string, any>, key: string) {
+    if (key.includes('.')) {
+        const keys = key.split('.');
+        let value = obj;
+        for (const k of keys) {
+            value = value[k];
+            if (value == undefined) {
+                return undefined;
+            }
+        }
+        return value;
+    } else {
+        return obj[key];
+    }
+}