@arkadia/ai-data-format 1.0.0

package/tests/00.primitive.test.ts

@@ -0,0 +1,108 @@
+import { describe, it, expect } from 'vitest';
+import { encode, decode, MetaInfo, parse } from '../src/index';
+import { assertRoundtrip } from './utils';
+
+// ==================================================================================
+// 1. DECODING TESTS (String -> Node)
+// ==================================================================================
+
+describe('AI Data Primitives', () => {
+    it('should encode raw object', () => {
+        const input = { foo: "bar" };
+        const expected = '<foo:string>("bar")';
+        assertRoundtrip(input, expected);
+    });
+
+    it.each([
+        // [Input Text, Expected Value, Expected Encoded Output]
+        ["123", 123, "<number>123"],
+        ["-50", -50, "<number>-50"],
+        ['"hello"', "hello", '<string>"hello"'],
+        ['"hello world"', "hello world", '<string>"hello world"'],
+        ["true", true, "<bool>true"],
+        ["false", false, "<bool>false"],
+        ["null", null, "<null>null"],
+    ])('should decode and encode primitive: %s', (text, expectedVal, expectedEnc) => {
+        // 1. DECODE
+        // We pass an options object (assuming the latest signature: decode(text, { debug: false }))
+        const res = decode(text, { debug: false });
+        
+        expect(res.errors, `Parsing failed for input: ${text}`).toHaveLength(0);
+        expect(res.node.value).toBe(expectedVal);
+        expect(res.node.isPrimitive).toBe(true);
+
+        // 2. ENCODE
+        const encodedStr = encode(res.node, { colorize: false, compact: true }).trim();
+        
+        expect(encodedStr).toBe(expectedEnc);
+    });
+
+    it('should decode floats', () => {
+        const cases: [string, number][] = [
+            ["12.34", 12.34],
+            ["-0.005", -0.005],
+            ["0.0", 0.0]
+        ];
+
+        cases.forEach(([text, expected]) => {
+            const res = decode(text);
+            expect(res.errors).toHaveLength(0);
+            expect(res.node.value).toBe(expected);
+            expect(typeof res.node.value).toBe('number'); 
+        });
+    });
+
+    it('should decode named record', () => {
+        const text = '{id: 1, name: "Test"}';
+        const res = decode(text, { debug: false });
+        expect(res.errors).toHaveLength(0);
+        
+        const node = res.node;
+
+        expect(node.isRecord).toBe(true);
+        expect(node.fields["id"].value).toBe(1);
+        expect(node.fields["name"].value).toBe("Test");
+
+        assertRoundtrip(
+            node, 
+            '<id:number,name:string>(1,"Test")', 
+            false
+        );
+    });
+
+    it('should decode positional record', () => {
+        const text = '(10, "Alice")';
+        const res = decode(text, { debug: false });
+        expect(res.errors).toHaveLength(0);
+        
+        const node = res.node;
+
+        expect(node.isRecord).toBe(true);
+        
+        // Verify if the parser mapped positional fields to _0, _1
+        expect(node.fields["_0"].value).toBe(10);
+        expect(node.fields["_1"].value).toBe("Alice");
+
+        assertRoundtrip(
+            node, 
+            '<_0:number,_1:string>(10,"Alice")',
+            false
+        );
+    });
+
+    it('should decode raw string', () => {
+        const text = "{color: red, status: active}";
+        const res = decode(text, { debug: false });
+        expect(res.errors).toHaveLength(0);
+        
+        const node = res.node;
+        expect(node.fields["color"].value).toBe("red");
+        expect(node.fields["status"].value).toBe("active");
+
+        assertRoundtrip(
+            node, 
+            '<color:string,status:string>("red","active")',
+            false
+        );
+    });
+});

package/tests/00.schema.test.ts

@@ -0,0 +1,41 @@
+import { describe, it, expect } from 'vitest';
+import { Schema, SchemaKind } from '../src/index';
+
+describe('AI Schema test', () => {
+    it('should encode raw object', () => {
+        const schema = new Schema(
+            SchemaKind.DICT,
+            {
+                name: "TestSchema",
+                comments: ["This is a comment"],
+                attr: { foo: "bar" },
+                tags: ["tag1", "tag2"],
+                required: true
+
+            });
+        const expected = '<Schema(DICT) name="TestSchema" !required attr=["foo"] tags=[tag1, tag2] comments=1>';
+        const expected_val = {
+            "comments": [
+                "This is a comment"
+            ],
+            "attr": {
+                "foo": "bar"
+            },
+            "tags": [
+                "tag1",
+                "tag2"
+            ],
+            "_fieldsList": [],
+            "_fieldsMap": {},
+            "kind": "DICT",
+            "typeName": "any",
+            "name": "TestSchema",
+            "element": null,
+            "key": null,
+            "value": null,
+            "required": true
+        }
+        expect(schema).toMatchObject(expected_val);
+        expect(schema.toString()).toBe(expected);
+    });
+});

package/tests/01.schema.test.ts

@@ -0,0 +1,70 @@
+import { describe, it, expect } from 'vitest';
+import { decode } from '../src/index';
+import { assertRoundtrip } from './utils';
+
+// ==================================================================================
+// 2. SCHEMA DEFINITION & TYPING
+// ==================================================================================
+
+describe('AI Data Schema Definitions', () => {
+
+    it('should handle schema definition and usage', () => {
+        /**
+         * Validates that a type defined with @Type<...> is correctly applied
+         * to the following value.
+         */
+        // Define schema first, then use it explicitly
+        const fullText = '@User<id:int, name:string> @User(1, "Admin")';
+        const res = decode(fullText, { debug: false });
+
+        expect(res.errors).toHaveLength(0);
+        
+        const node = res.node;
+        // Check if schema is linked correctly
+        expect(node.schema).not.toBeNull();
+        expect(node.schema?.typeName).toBe("User");
+
+        // Since we have a schema, positional arguments should be mapped to fields
+        // Check by key (Decoder maps positional to fields if schema exists)
+        expect(node.fields["id"].value).toBe(1);
+        expect(node.fields["name"].value).toBe("Admin");
+
+        assertRoundtrip(
+            node, 
+            '@User<id:number,name:string>(1,"Admin")',
+            false
+        );
+    });
+
+    it('should handle nested schema structure', () => {
+        /**
+         * Validates nested structural types.
+         */
+        // We define Profile, then User uses it.
+        const text = `
+        @Profile<level:int>
+        @User<id:int, profile: @Profile>
+        @User(1, {level: 99})
+        `;
+        
+        const res = decode(text, { debug: false });
+
+        expect(res.errors).toHaveLength(0);
+        const node = res.node;
+
+        // id should be 1
+        expect(node.fields["id"].value).toBe(1);
+
+        // profile should be a node
+        const profileNode = node.fields["profile"];
+        expect(profileNode).toBeDefined();
+        expect(profileNode.fields["level"].value).toBe(99);
+
+        assertRoundtrip(
+            node, 
+            "@User<id:number,profile:@Profile<level:number>>(1,(99))",
+            false
+        );
+    });
+
+});

package/tests/02.data.test.ts

@@ -0,0 +1,89 @@
+import { describe, it, expect } from 'vitest';
+import { encode, decode } from '../src/index';
+import { assertRoundtrip } from './utils';
+
+// ==================================================================================
+// 3. ENCODING TESTS (JS Object -> String)
+// ==================================================================================
+
+describe('AI Data Encoding', () => {
+
+    it('should encode simple dict', () => {
+        /**
+         * Validates encoding a JS Object to AI.DATA format.
+         */
+        const data = { x: 10, y: 20 };
+        
+        const result = encode(data, { compact: true });
+        const expected = '<x:number,y:number>(10,20)';
+        
+        expect(result).toBe(expected);
+        
+        assertRoundtrip(
+            result, 
+            expected, 
+            false
+        );
+    });
+
+    it('should encode list of objects', () => {
+        /**
+         * Validates encoding a list of objects.
+         */
+        const data = [{ name: "A", val: 1 }, { name: "B", val: 2 }];
+        
+        const result = encode(data, { compact: true, colorize: false });
+        const expected = '<[name:string,val:number]>[("A",1),("B",2)]';
+        
+        expect(result).toBe(expected);
+
+        assertRoundtrip(
+            result, 
+            expected,
+            false
+        );
+    });
+
+    it('should handle round trip consistency', () => {
+        /**
+         * Golden Test: Encode -> Decode -> Compare.
+         */
+        const originalData = [
+            { id: 1, active: true, tags: ["a", "b"] },
+            { id: 2, active: false, tags: ["c"] },
+        ];
+        
+        const expected = '<[id:number,active:bool,tags:[string]]>[(1,true,["a","b"]),(2,false,["c"])]';
+
+        // 1. Encode
+        // Note: converted snake_case config keys to camelCase for TS
+        const encodedText = encode(
+            originalData,
+            { compact: true, includeSchema: true, colorize: false }
+        );
+
+        expect(encodedText).toBe(expected);
+        assertRoundtrip(encodedText, expected, false);
+
+        // 2. Decode
+        const res = decode(encodedText, { debug: false });
+        
+        // Optional: inspect schema structure
+
+        expect(res.errors).toHaveLength(0);
+
+        // 3. Convert back to JS object (equivalent to .dict() in Python)
+        const decodedData = res.node.dict()
+
+        // 4. Compare
+        expect(decodedData).toHaveLength(2);
+        expect(decodedData[0].id).toBe(1);
+        expect(decodedData[0].active).toBe(true);
+        expect(decodedData[0].tags).toEqual(["a", "b"]);
+        expect(decodedData[1].active).toBe(false);
+
+        // Deep equality check
+        expect(decodedData).toEqual(originalData);
+    });
+
+});

package/tests/03.errors.test.ts

@@ -0,0 +1,71 @@
+import { describe, it, expect } from 'vitest';
+import { encode, decode } from '../src/index';
+
+// ==================================================================================
+// 4. ROUND TRIP (Encode -> Decode) with Errors
+// ==================================================================================
+
+describe('A Data Error Handling', () => {
+
+    it('should report errors on unstripped ANSI codes', () => {
+        /**
+         * Validates that the Decoder handles ANSI codes gracefully (by reporting errors)
+         * if they are not stripped out before decoding.
+         */
+        const originalData = [{ id: 1, active: true }, { id: 2, active: false }];
+
+        // 1. Encode WITH colors
+        // Note: We use camelCase keys for config in TypeScript
+        const encodedText = encode(
+            originalData,
+            {
+                compact: true,
+                includeSchema: true,
+                colorize: true,
+            }
+        );
+
+        // 2. Decode WITHOUT stripping colors (simulate user error)
+        // Pass removeAnsiColors: false explicitely
+        const res = decode(encodedText, { removeAnsiColors: false, debug: false });
+
+        // 3. Assertions
+        expect(res.errors.length).toBeGreaterThan(0); // Decoder should report errors on raw ANSI codes
+        
+        // Optional: Verify error content
+        const messages = res.errors.map(e => e.message);
+        const hasUnexpectedChar = messages.some(m => m.includes("Unexpected character"));
+        expect(hasUnexpectedChar).toBe(true);
+    });
+
+    // ==================================================================================
+    // 5. PARSING ERROR SCENARIOS
+    // ==================================================================================
+
+    it('should fail on unclosed list', () => {
+        /**
+         * Ensures parsing fails for malformed lists.
+         */
+        const text = "[1, 2, 3";  // Missing closing bracket
+        const res = decode(text, { debug: false });
+        
+        expect(res.errors.length).toBeGreaterThan(0);
+        
+        const msg = res.errors[0].message;
+        // Check for typical error messages related to missing tokens or EOF
+        const isExpectedError = msg.includes("Expected") || msg.includes("got") || msg.includes("EOF");
+        expect(isExpectedError).toBe(true);
+    });
+
+    it('should fail on unexpected character', () => {
+        /**
+         * Ensures parsing fails for illegal characters.
+         */
+        const text = "(1, ?)";
+        const res = decode(text, { debug: false });
+        
+        expect(res.errors.length).toBeGreaterThan(0);
+        expect(res.errors[0].message).toContain("Unexpected character");
+    });
+
+});

package/tests/04.list.test.ts

@@ -0,0 +1,225 @@
+import { describe, it, expect } from 'vitest';
+import { encode, decode, parse, SchemaKind } from '../src/index';
+import { assertRoundtrip } from './utils';
+
+describe('AI Data List Handling', () => {
+
+    it('should decode list of primitives', () => {
+        /**
+         * Validates simple lists.
+         */
+        const text = "[1, 2, 3]";
+        const expected = "<[number]>[1,2,3]";
+        
+        const res = decode(text, { debug: false });
+        expect(res.errors).toHaveLength(0);
+
+        const node = res.node;
+        // Check elements (Node logic stores simple lists in elements)
+        const values = node.elements.map(el => el.value);
+        expect(values).toEqual([1, 2, 3]);
+        
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should infer mixed list type from first element', () => {
+        /**
+         * Tests if the list type is inferred based on the first element ("a" -> string).
+         * As a result, the number 3 (int) should be treated as a mismatch and marked with the <number> tag.
+         */
+        // 1. Input data: List starts with strings but has an int at the end
+        const data = { tests: ["a", "b", "c", 3] };
+
+        // 2. Encoding (inference happens here in Parser.ts)
+        const node = parse(data);
+
+        expect(node.schema).not.toBeNull();
+        expect(node.schema.isRecord).toBe(true);
+        expect(node.fields).toHaveProperty("tests");
+        
+        const testsNode = node.fields["tests"];
+        expect(testsNode.isList).toBe(true);
+        expect(testsNode.schema).not.toBeNull();
+        expect(testsNode.schema.element).not.toBeNull();
+
+        expect(testsNode.schema.element?.kind).toBe(SchemaKind.PRIMITIVE);
+        expect(testsNode.schema.element?.typeName).toBe("string"); // Inferred from first element "a"
+
+        expect(testsNode.elements).toHaveLength(4);
+
+        const output = encode(data, { compact: true, colorize: false });
+
+        // We expect the list NOT to be [any], but [string] (implied or explicit),
+        // so strings will be "clean", and the number will get a tag.
+
+        // Check if 'a' is treated normally (as a string in a string list)
+        expect(output).toContain('"a"');
+
+        // KEY: Check if 3 got a tag because it doesn't match the inferred String type
+        // TS Encoder outputs <number>
+        expect(output).toContain("<number> 3");
+
+        // Ensure there is NO tag next to strings (because they match the list type)
+        expect(output).not.toContain('<string> "a"');
+
+        const expected = '<tests:[string]>(["a","b","c",<number> 3])';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should generate tags for explicit any list due to primitive mismatch', () => {
+        /**
+         * Tests the scenario where a list is defined as [any].
+         * * Behavior:
+         * 1. 'any' is parsed as a PRIMITIVE type named "any".
+         * 2. The Decoder updates the schema based on the first element found ("a" -> string).
+         * 3. Because "string" != "number" (for value 3), the Encoder sees a mismatch and adds explicit tags.
+         */
+
+        // 1. Input in AID format
+        const aidText = `
+        <tests: [any]>
+        (
+            ["a", "b", "c", 3]
+        )
+        `;
+
+        // 2. Decode
+        const result = decode(aidText, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        // 3. Verify Internal State
+        const testsNode = node.fields["tests"];
+
+        // Verify the list definition
+        const elementSchema = testsNode.schema.element;
+        expect(elementSchema?.kind).toBe(SchemaKind.PRIMITIVE);
+        
+        // Decoder logic updates "any" element schema based on the first item found
+        expect(elementSchema?.typeName).toBe("string");
+
+        // Verify the actual elements
+        expect(testsNode.elements[0].schema.typeName).toBe("string");
+        expect(testsNode.elements[3].schema.typeName).toBe("number");
+
+        // 4. Encode
+        const expected = '<tests:[string]>(["a","b","c",<number> 3])';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle inference happy path', () => {
+        /**
+         * Theory: If a list has no type (or [any]), the first element ("a")
+         * should refine the list schema to [string].
+         */
+        const dataStr = '["a", "b"]';  // No header = SchemaKind.ANY
+
+        const result = decode(dataStr, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        expect(node.isList).toBe(true);
+        expect(node.schema.element?.typeName).toBe("string"); // Inferred!
+
+        const expected = '<[string]>["a","b"]';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle inference mismatch (string vs number)', () => {
+        /**
+         * Theory: First element "a" sets list to [string].
+         * The number 3 is mismatch and gets tagged.
+         */
+        const dataStr = '["a", 3]';
+
+        const result = decode(dataStr, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        // The list should have become [string] due to "a"
+        expect(node.schema.element?.typeName).toBe("string");
+
+        const expected = '<[string]>["a",<number> 3]';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle inference mismatch (number vs string)', () => {
+        /**
+         * Theory: First element 3 sets list to [number].
+         * The string "a" is mismatch and gets tagged.
+         */
+        const dataStr = '[3, "a"]';
+
+        const result = decode(dataStr, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        // The list should have become [number] due to 3
+        expect(node.schema.element?.typeName).toBe("number");
+
+        const expected = '<[number]>[3,<string> "a"]';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should fix schema crash on override logic', () => {
+        /**
+         * Theory: We expect a String, but we get a List.
+         * This triggers 'needs_override' logic in the Encoder.
+         */
+        // Header says 'test' is a string, but body has a list
+        const aidText = `
+        <test: string>
+        (
+            ["a", "b"]
+        )
+        `;
+
+        const result = decode(aidText, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        // The Encoder sees a List Node vs String Schema -> Mismatch -> Adds Tag
+        const expected = '<test:string>(<[string]> ["a","b"])';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle primitive list with implicit output', () => {
+        const aidText = `
+         <ab>
+        {
+            ab:  ["a", "b", "c", 3]
+        }
+        `;
+
+        const result = decode(aidText, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        const expected = '<ab:[string]>(["a","b","c",<number> 3])';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle simple mixed types', () => {
+        const aidText = '["a", "b", "c", 3]';
+
+        const result = decode(aidText, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        const expected = '<[string]>["a","b","c",<number> 3]';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle inner list types', () => {
+        // <[[int]]>[[2,3,4],[5,6,7]]
+        const aidText = '<[[int]]>[[2,3,4],[5,6,7]]';
+
+        const result = decode(aidText, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+
+        // 'int' normalizes to 'number' in the TS implementation
+        const expected = '<[[number]]>[[2,3,4],[5,6,7]]';
+        assertRoundtrip(node, expected, false);
+    });
+});

package/tests/05.record.test.ts

@@ -0,0 +1,82 @@
+import { describe, it, expect } from 'vitest';
+import { decode } from '../src/index';
+import { assertRoundtrip } from './utils';
+
+describe('AI Data Record Handling', () => {
+
+    // ==============================================================================
+    // 1. SCHEMA DEFINITION META (Types defined in < ... >)
+    // ==============================================================================
+
+    it('should handle different type in record', () => {
+        const aidText = `
+        < 
+          id: number
+        >
+        ( ["text"] )
+        `;
+
+        const result = decode(aidText, { debug: false });
+        const node = result.node;
+        expect(result.errors).toHaveLength(0);
+        
+        // 1. Check Record Meta (Outer)
+        expect(node.isRecord).toBe(true);
+
+        const expected = '<id:number>(<[string]> ["text"])';
+        assertRoundtrip(node, expected, false);
+    });
+
+    it('should handle simple types', () => {
+        // Input is a raw AID format string representing a dictionary
+        const aidText = '{ a:"a", b:"b", c:"c", d: 3 }';
+        
+        // The parser infers the schema from the keys/values.
+        // The encoder then outputs the inferred schema and converts the named record 
+        // to a positional one because a schema exists.
+        const expected = '<a:string,b:string,c:string,d:number>("a","b","c",3)';
+        
+        assertRoundtrip(aidText, expected, false);   
+    });
+
+    it('should handle record named type mismatch', () => {
+        /**
+         * Tests a scenario where a record field has a defined type (e.g., string),
+         * but the actual value inside is of a different type (e.g., int).
+         *
+         * This ensures that the encodeRecord method uses applyTypeTag correctly.
+         *
+         * Schema: <tests: string>
+         * Data: { tests: 3 }
+         * Expected Output: ... (<number> 3)
+         */
+        const aidText = `
+        <tests: string>
+        {
+         tests: 3
+        }
+        `;
+
+        const expected = '<tests:string>(<number> 3)';
+        assertRoundtrip(aidText, expected, false);   
+    });
+
+    it('should handle record positional type mismatch', () => {
+        /**
+         * Tests a scenario where a positional record field has a defined type (e.g., string),
+         * but the actual value inside is of a different type (e.g., int).
+         *
+         * Schema: <tests: string>
+         * Data: (3)
+         * Expected Output: ... (<number> 3)
+         */
+        const aidText = `
+        <tests: string>
+        (3)
+        `;
+
+        const expected = '<tests:string>(<number> 3)';
+        assertRoundtrip(aidText, expected, false);   
+    });
+
+});