@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/memory/observation.js

@@ -0,0 +1,79 @@
+/**
+ * Observations and declarations for what kind of data is "observable" - i.e. can be processed into BAML context or JSON.
+ */
+import { fnv1a32Hex } from '@/util';
+import { observableDataToJson } from './serde';
+import { renderContentParts } from './rendering';
+export class Observation {
+    source; // where this observation came from
+    role;
+    timestamp; // time this observation was made
+    content; // the arbitrarily-structured multimedia content of this observation
+    retention;
+    constructor(source, role, content, retention, timestamp) {
+        this.source = source;
+        this.role = role;
+        this.content = content;
+        this.timestamp = timestamp ?? Date.now();
+        this.retention = retention;
+    }
+    static fromConnector(connectorId, content, options) {
+        return new Observation(`connector:${connectorId}`, 'user', content, options);
+    }
+    static fromActionTaken(actionId, content, options) {
+        // see notes on fromThought
+        return new Observation(`action:taken:${actionId}`, 'user', content, options);
+    }
+    static fromActionResult(actionId, content, options) {
+        return new Observation(`action:result:${actionId}`, 'user', content, options);
+    }
+    static fromThought(content, options) {
+        // may want to reconsider this structure - probably want to show something that actually matches what the assistant sent else it might pattern match.
+        // or just do these as user messages.
+        // changed to user for now
+        return new Observation(`thought`, 'user', content, options);
+    }
+    toString() {
+        // will be pretty ugly usually
+        return JSON.stringify(this.content);
+    }
+    async toJson() {
+        return await observableDataToJson(this.content);
+    }
+    async render(options) {
+        return {
+            role: this.role,
+            cacheControl: options?.cacheControl ?? false,
+            content: [...(options?.prefix ?? []), ...(await renderContentParts(this.content, { mode: 'json', indent: 2 })), ...(options?.postfix ?? [])]
+        };
+    }
+    // async renderContentParts(): Promise<MultiMediaContentPart[]> {
+    //     return await renderParts(this.content);
+    // }
+    // async toContext(): Promise<BamlRenderable[]> {
+    //     return await observableDataToContext(this.data);
+    // }
+    async hash() {
+        // build str of all text+img content
+        const stringifiedContent = JSON.stringify(await this.toJson());
+        return fnv1a32Hex(stringifiedContent);
+    }
+    async equals(obs) {
+        // TODO: deep eq for both text and img content
+        return (await this.hash()) == (await obs.hash());
+    }
+}
+// export async function renderObservations(observations: Observation[]): Promise<BamlRenderable[]> {
+//     /**
+//      * Refines observations (according to observation culling options) and converts to BAML-renderable content
+//      */
+//     const filteredObservations = await filterObservations(observations);
+//     let content: BamlRenderable[] = [];
+//     for (const obs of filteredObservations) {
+//         if (obs.source.startsWith('action:taken') || obs.source.startsWith('thought')) {
+//             content.push(`[${new Date(obs.timestamp).toTimeString().split(' ')[0]}]: `);
+//         }
+//         content = [...content, ...await obs.toContext(), "\n"];
+//     }
+//     return content;
+// }

package/dist/memory/rendering/index.d.ts

@@ -0,0 +1,10 @@
+import { RenderableContent } from "@/memory/observation";
+import { Image as BamlImage } from '@boundaryml/baml';
+export type MultiMediaContentPart = BamlImage | string;
+export type RenderOptions = {
+    mode: 'json';
+    indent: 0 | 2 | 4;
+} | {
+    mode: 'xml';
+};
+export declare function renderContentParts(data: RenderableContent, options: RenderOptions): Promise<import("./renderXmlParts").MultiMediaContentPart[]>;

package/dist/memory/rendering/index.js

@@ -0,0 +1,13 @@
+import { renderXmlParts } from "./renderXmlParts";
+import { renderJsonParts } from "./renderJsonParts";
+export async function renderContentParts(data, options) {
+    if (options.mode === 'json') {
+        return await renderJsonParts(data, options.indent);
+    }
+    else if (options.mode === 'xml') {
+        return await renderXmlParts(data);
+    }
+    else {
+        throw new Error(`Invalid render mode ${options.mode}`);
+    }
+}

package/dist/memory/rendering/renderJsonParts.d.ts

@@ -0,0 +1,3 @@
+import { RenderableContent } from "@/memory/observation";
+import { MultiMediaContentPart } from "@/memory/rendering";
+export declare function renderJsonParts(data: RenderableContent, indent: number): Promise<MultiMediaContentPart[]>;

package/dist/memory/rendering/renderJsonParts.js

@@ -0,0 +1,115 @@
+import { Image } from "@/memory/image";
+import { Image as BamlImage } from '@boundaryml/baml';
+async function buildJsonPartsRecursive(data, partsList, indent, currentLevel = 0, isRoot = false) {
+    if (data instanceof Image) {
+        const bamlImg = await data.toBaml();
+        if (bamlImg) {
+            partsList.push(bamlImg);
+        }
+        return;
+    }
+    if (data instanceof BamlImage) {
+        partsList.push(data);
+        return;
+    }
+    if (data === null) {
+        partsList.push('null');
+        return;
+    }
+    if (data === undefined) {
+        return; // Undefined values are omitted in JSON
+    }
+    if (typeof data === 'string') {
+        if (isRoot) {
+            partsList.push(data);
+        }
+        else {
+            partsList.push(JSON.stringify(data));
+        }
+        return;
+    }
+    if (typeof data === 'number' || typeof data === 'boolean') {
+        partsList.push(String(data));
+        return;
+    }
+    const newline = indent > 0 ? '\n' : '';
+    const spacing = indent > 0 ? ' '.repeat(currentLevel * indent) : '';
+    const nextSpacing = indent > 0 ? ' '.repeat((currentLevel + 1) * indent) : '';
+    if (Array.isArray(data)) {
+        partsList.push('[');
+        for (let index = 0; index < data.length; index++) {
+            const item = data[index];
+            if (item !== undefined) { // Skip undefined values in arrays
+                if (index > 0) {
+                    partsList.push(',');
+                }
+                if (indent > 0) {
+                    partsList.push(newline + nextSpacing);
+                }
+                else if (index > 0) {
+                    partsList.push(' ');
+                }
+                await buildJsonPartsRecursive(item, partsList, indent, currentLevel + 1, false);
+            }
+        }
+        if (data.length > 0 && indent > 0) {
+            partsList.push(newline + spacing);
+        }
+        partsList.push(']');
+        return;
+    }
+    if (typeof data === 'object' && data !== null) {
+        partsList.push('{');
+        const entries = Object.entries(data).filter(([, val]) => val !== undefined);
+        for (let index = 0; index < entries.length; index++) {
+            const [key, value] = entries[index];
+            if (index > 0) {
+                partsList.push(',');
+            }
+            if (indent > 0) {
+                partsList.push(newline + nextSpacing);
+            }
+            else if (index > 0) {
+                partsList.push(' ');
+            }
+            // Add the key
+            partsList.push(JSON.stringify(key));
+            partsList.push(': ');
+            // Add the value
+            await buildJsonPartsRecursive(value, partsList, indent, currentLevel + 1, false);
+        }
+        if (entries.length > 0 && indent > 0) {
+            partsList.push(newline + spacing);
+        }
+        partsList.push('}');
+        return;
+    }
+    throw new Error(`Object type not supported for JSON rendering: '${typeof data}'. Data: ${data}`);
+}
+export async function renderJsonParts(data, indent) {
+    const rawList = [];
+    await buildJsonPartsRecursive(data, rawList, indent, 0, true);
+    // Merge adjacent strings
+    if (rawList.length === 0) {
+        return [];
+    }
+    const mergedList = [];
+    let currentString = "";
+    for (const item of rawList) {
+        if (typeof item === 'string') {
+            currentString += item;
+        }
+        else { // It's a BamlImage
+            if (currentString.length > 0) {
+                mergedList.push(currentString);
+            }
+            currentString = ""; // Reset accumulator
+            mergedList.push(item); // Push the BamlImage
+        }
+    }
+    // Add any trailing string
+    if (currentString.length > 0) {
+        mergedList.push(currentString);
+    }
+    return mergedList;
+}

package/dist/memory/rendering/renderJsonParts.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/memory/rendering/renderJsonParts.test.js

@@ -0,0 +1,217 @@
+import { describe, expect, test } from 'bun:test';
+import { renderJsonParts } from './renderJsonParts';
+import { Image } from '@/memory/image';
+// Create a mock Image that will convert to BamlImage
+class MockImage extends Image {
+    id;
+    constructor(id) {
+        // Pass a dummy Sharp instance
+        super(null);
+        this.id = id;
+    }
+    async toBaml() {
+        // Return a mock BamlImage
+        return {
+            __mockBamlImage: true,
+            id: this.id
+        };
+    }
+}
+describe('renderJsonParts', () => {
+    test('renders primitive types correctly', async () => {
+        expect(await renderJsonParts('hello', 0)).toEqual(['hello']);
+        expect(await renderJsonParts(42, 0)).toEqual(['42']);
+        expect(await renderJsonParts(true, 0)).toEqual(['true']);
+        expect(await renderJsonParts(false, 0)).toEqual(['false']);
+        expect(await renderJsonParts(null, 0)).toEqual(['null']);
+    });
+    test('renders root-level strings without quotes', async () => {
+        expect(await renderJsonParts('CLICK THE UNICORN', 0)).toEqual(['CLICK THE UNICORN']);
+        expect(await renderJsonParts('Hello World', 0)).toEqual(['Hello World']);
+        expect(await renderJsonParts('{"already":"json"}', 0)).toEqual(['{"already":"json"}']);
+    });
+    test('properly quotes strings inside objects and arrays', async () => {
+        // Strings in objects should be quoted
+        expect(await renderJsonParts({ message: 'hello' }, 0)).toEqual(['{"message": "hello"}']);
+        expect(await renderJsonParts({ name: 'John', city: 'NYC' }, 0)).toEqual(['{"name": "John", "city": "NYC"}']);
+        // Strings in arrays should be quoted
+        expect(await renderJsonParts(['apple', 'banana'], 0)).toEqual(['["apple", "banana"]']);
+        expect(await renderJsonParts(['hello', 123, 'world'], 0)).toEqual(['["hello", 123, "world"]']);
+        // Nested structures
+        expect(await renderJsonParts({ items: ['a', 'b'], text: 'test' }, 0)).toEqual(['{"items": ["a", "b"], "text": "test"}']);
+    });
+    test('renders empty structures', async () => {
+        expect(await renderJsonParts([], 0)).toEqual(['[]']);
+        expect(await renderJsonParts({}, 0)).toEqual(['{}']);
+    });
+    test('renders simple arrays', async () => {
+        expect(await renderJsonParts([1, 2, 3], 0)).toEqual(['[1, 2, 3]']);
+        expect(await renderJsonParts(['a', 'b', 'c'], 0)).toEqual(['["a", "b", "c"]']);
+        expect(await renderJsonParts([true, false, null], 0)).toEqual(['[true, false, null]']);
+    });
+    test('renders simple objects', async () => {
+        expect(await renderJsonParts({ name: 'John', age: 30 }, 0)).toEqual(['{"name": "John", "age": 30}']);
+        expect(await renderJsonParts({ a: 1, b: true, c: null }, 0)).toEqual(['{"a": 1, "b": true, "c": null}']);
+    });
+    test('handles undefined values correctly', async () => {
+        // Undefined at root level returns empty array
+        expect(await renderJsonParts(undefined, 0)).toEqual([]);
+        // Undefined in arrays is skipped
+        expect(await renderJsonParts([1, undefined, 3], 0)).toEqual(['[1, 3]']);
+        // Undefined in objects is omitted
+        expect(await renderJsonParts({ a: 1, b: undefined, c: 3 }, 0)).toEqual(['{"a": 1, "c": 3}']);
+    });
+    test('renders nested structures', async () => {
+        const nested = {
+            user: {
+                name: 'Alice',
+                scores: [95, 87, 92],
+                active: true
+            },
+            metadata: {
+                created: '2024-01-01',
+                tags: ['important', 'verified']
+            }
+        };
+        const result = await renderJsonParts(nested, 0);
+        expect(result).toEqual(['{"user": {"name": "Alice", "scores": [95, 87, 92], "active": true}, "metadata": {"created": "2024-01-01", "tags": ["important", "verified"]}}']);
+    });
+    test('splits JSON around BamlImage in object', async () => {
+        const mockImage = new MockImage('img1');
+        const data = {
+            title: 'Product',
+            image: mockImage,
+            price: 29.99
+        };
+        const result = await renderJsonParts(data, 0);
+        expect(result.length).toBe(3);
+        expect(result[0]).toBe('{"title": "Product", "image": ');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(', "price": 29.99}');
+    });
+    test('splits JSON around multiple BamlImages in object', async () => {
+        const mockImage1 = new MockImage('img1');
+        const mockImage2 = new MockImage('img2');
+        const data = {
+            before: 'text',
+            image1: mockImage1,
+            middle: 'more text',
+            image2: mockImage2,
+            after: 'final'
+        };
+        const result = await renderJsonParts(data, 0);
+        expect(result.length).toBe(5);
+        expect(result[0]).toBe('{"before": "text", "image1": ');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(', "middle": "more text", "image2": ');
+        expect(result[3]).toHaveProperty('__mockBamlImage', true);
+        expect(result[4]).toBe(', "after": "final"}');
+    });
+    test('splits JSON around BamlImage in array', async () => {
+        const mockImage = new MockImage('img1');
+        const data = ['start', mockImage, 'end'];
+        const result = await renderJsonParts(data, 0);
+        expect(result.length).toBe(3);
+        expect(result[0]).toBe('["start", ');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(', "end"]');
+    });
+    test('handles array with only BamlImages', async () => {
+        const mockImage1 = new MockImage('img1');
+        const mockImage2 = new MockImage('img2');
+        const data = [mockImage1, mockImage2];
+        const result = await renderJsonParts(data, 0);
+        expect(result.length).toBe(5);
+        expect(result[0]).toBe('[');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(', ');
+        expect(result[3]).toHaveProperty('__mockBamlImage', true);
+        expect(result[4]).toBe(']');
+    });
+    test('handles nested structures with images', async () => {
+        const mockImage = new MockImage('img1');
+        const data = {
+            user: {
+                name: 'Bob',
+                avatar: mockImage,
+                settings: {
+                    theme: 'dark'
+                }
+            },
+            posts: [
+                { id: 1, image: mockImage, text: 'Hello' }
+            ]
+        };
+        const result = await renderJsonParts(data, 0);
+        // Should have multiple parts due to images
+        expect(result.length).toBeGreaterThan(1);
+        // First part should start with JSON
+        expect(result[0]).toMatch(/^{"user": /);
+        // Should contain the mock images
+        const imageParts = result.filter(part => part && typeof part === 'object' && '__mockBamlImage' in part);
+        expect(imageParts.length).toBe(2);
+    });
+    test('handles edge case with image as first array element', async () => {
+        const mockImage = new MockImage('img1');
+        const data = [mockImage, 'text'];
+        const result = await renderJsonParts(data, 0);
+        expect(result.length).toBe(3);
+        expect(result[0]).toBe('[');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(', "text"]');
+    });
+    test('handles edge case with image as last array element', async () => {
+        const mockImage = new MockImage('img1');
+        const data = ['text', mockImage];
+        const result = await renderJsonParts(data, 0);
+        expect(result.length).toBe(3);
+        expect(result[0]).toBe('["text", ');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(']');
+    });
+    test('properly escapes special characters in strings', async () => {
+        const data = {
+            quote: 'He said "Hello"',
+            newline: 'Line 1\nLine 2',
+            tab: 'Col1\tCol2',
+            backslash: 'C:\\Users\\Path'
+        };
+        const result = await renderJsonParts(data, 0);
+        expect(result[0]).toContain('\\"Hello\\"');
+        expect(result[0]).toContain('\\n');
+        expect(result[0]).toContain('\\t');
+        expect(result[0]).toContain('\\\\');
+    });
+    test('renders with proper indentation when indent is 2', async () => {
+        const data = {
+            name: 'John',
+            age: 30,
+            hobbies: ['reading', 'coding']
+        };
+        const result = await renderJsonParts(data, 2);
+        expect(result).toEqual(['{\n  "name": "John",\n  "age": 30,\n  "hobbies": [\n    "reading",\n    "coding"\n  ]\n}']);
+    });
+    test('renders with proper indentation when indent is 4', async () => {
+        const data = {
+            user: {
+                name: 'Alice',
+                active: true
+            }
+        };
+        const result = await renderJsonParts(data, 4);
+        expect(result).toEqual(['{\n    "user": {\n        "name": "Alice",\n        "active": true\n    }\n}']);
+    });
+    test('handles indentation with images', async () => {
+        const mockImage = new MockImage('img1');
+        const data = {
+            title: 'Product',
+            image: mockImage,
+            price: 29.99
+        };
+        const result = await renderJsonParts(data, 2);
+        expect(result.length).toBe(3);
+        expect(result[0]).toBe('{\n  "title": "Product",\n  "image": ');
+        expect(result[1]).toHaveProperty('__mockBamlImage', true);
+        expect(result[2]).toBe(',\n  "price": 29.99\n}');
+    });
+});

package/dist/memory/rendering/renderXmlParts.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Convert observations to form that can be passed as BAML context, rendered as a custom XML-like structure.
+ */
+import { Image as BamlImage } from '@boundaryml/baml';
+import { RenderableContent } from '@/memory/observation';
+export type MultiMediaContentPart = BamlImage | string;
+export declare function renderXmlParts(data: RenderableContent): Promise<MultiMediaContentPart[]>;

package/dist/memory/rendering/renderXmlParts.js

@@ -0,0 +1,127 @@
+/**
+ * Convert observations to form that can be passed as BAML context, rendered as a custom XML-like structure.
+ */
+import { Image as BamlImage } from '@boundaryml/baml';
+import { Image } from '@/memory/image';
+//export type BamlRenderable = BamlImage | string;
+// export type BamlRenderable = {
+//     role: ''
+//     content: (BamlImage | string)[]
+// }
+async function buildXmlPartsRecursive(data, indentLevel, partsList, isInsideList = false) {
+    const indent = '  '.repeat(indentLevel);
+    if (data instanceof Image) {
+        const bamlImg = await data.toBaml();
+        if (bamlImg) {
+            partsList.push(bamlImg);
+        }
+        return;
+    }
+    if (data instanceof BamlImage) {
+        partsList.push(data);
+        return;
+    }
+    if (typeof data === 'string' || typeof data === 'number' || typeof data === 'boolean' || data === null) {
+        partsList.push(String(data)); // No XML escaping, direct string conversion
+        return;
+    }
+    if (data === undefined) {
+        return; // Undefined values are omitted
+    }
+    if (Array.isArray(data)) {
+        // data.forEach(async (item, index) => {
+        //     await buildXmlPartsRecursive(item, indentLevel, partsList, true);
+        //     if (index < data.length - 1) {
+        //         partsList.push("\n");
+        //     }
+        // });
+        for (let index = 0; index < data.length; index++) {
+            await buildXmlPartsRecursive(data[index], indentLevel, partsList, true);
+            if (index < data.length - 1) {
+                partsList.push("\n");
+            }
+        }
+        return;
+    }
+    if (typeof data === 'object' && data !== null) {
+        const objectEntries = Object.entries(data).filter(([, val]) => val !== undefined);
+        objectEntries.forEach(async ([key, value], entryIndex) => {
+            const tagName = key; // Use key directly as tag name
+            const currentValueParts = [];
+            await buildXmlPartsRecursive(value, indentLevel + 1, currentValueParts, false);
+            // Merge adjacent strings in currentValueParts for easier analysis
+            const mergedValueParts = [];
+            let currentStr = "";
+            for (const part of currentValueParts) {
+                if (typeof part === 'string') {
+                    currentStr += part;
+                }
+                else {
+                    if (currentStr)
+                        mergedValueParts.push(currentStr);
+                    currentStr = "";
+                    mergedValueParts.push(part);
+                }
+            }
+            if (currentStr)
+                mergedValueParts.push(currentStr);
+            // Apply styling rules
+            if (mergedValueParts.length === 1 && mergedValueParts[0] instanceof BamlImage) {
+                partsList.push(`${indent}<${tagName}>`);
+                partsList.push(mergedValueParts[0]); // The BamlImage
+                partsList.push(`</${tagName}>`);
+            }
+            else if (mergedValueParts.length === 1 && typeof mergedValueParts[0] === 'string') {
+                const contentStr = mergedValueParts[0];
+                if (contentStr.includes('\n')) {
+                    partsList.push(`${indent}<${tagName}>\n${contentStr}\n${indent}</${tagName}>`);
+                }
+                else {
+                    partsList.push(`${indent}<${tagName}>${contentStr}</${tagName}>`);
+                }
+            }
+            else { // Empty content or multiple parts (complex/nested)
+                partsList.push(`${indent}<${tagName}>\n`);
+                mergedValueParts.forEach((part, partIdx) => {
+                    partsList.push(part);
+                    if (partIdx < mergedValueParts.length - 1) {
+                        partsList.push("\n");
+                    }
+                });
+                partsList.push(`\n${indent}</${tagName}>`);
+            }
+            if (entryIndex < objectEntries.length - 1) {
+                partsList.push("\n"); // Newline between sibling XML elements
+            }
+        });
+        return;
+    }
+    throw new Error(`Object type not supported for LLM context: ${typeof data}`);
+}
+export async function renderXmlParts(data) {
+    const rawList = [];
+    await buildXmlPartsRecursive(data, 0, rawList);
+    // Merge adjacent strings
+    if (rawList.length === 0) {
+        return [];
+    }
+    const mergedList = [];
+    let currentString = "";
+    for (const item of rawList) {
+        if (typeof item === 'string') {
+            currentString += item;
+        }
+        else { // It's a BamlImage
+            if (currentString.length > 0) {
+                mergedList.push(currentString);
+            }
+            currentString = ""; // Reset accumulator
+            mergedList.push(item); // Push the BamlImage
+        }
+    }
+    // Add any trailing string
+    if (currentString.length > 0) {
+        mergedList.push(currentString);
+    }
+    return mergedList;
+}

package/dist/memory/serde.d.ts

@@ -0,0 +1,20 @@
+import { RenderableContent } from "./observation";
+export type StoredMedia = {
+    type: 'media';
+    format: string;
+    storage: 'base64';
+    base64: string;
+};
+export interface StoredPrimitive {
+    type: 'primitive';
+    content: string | boolean | number;
+}
+type MultiMediaPrimitive = StoredMedia | StoredPrimitive | undefined | null;
+type MultiMediaArray = Array<MultiMediaJson>;
+export type MultiMediaObject = {
+    [key: string]: MultiMediaJson;
+};
+export type MultiMediaJson = MultiMediaPrimitive | MultiMediaArray | MultiMediaObject;
+export declare function observableDataToJson(data: RenderableContent): Promise<MultiMediaJson>;
+export declare function jsonToObservableData(data: MultiMediaJson): Promise<RenderableContent>;
+export {};