@yeseh/cortex-cli 0.6.8 → 0.6.9

package/src/toon.spec.ts

@@ -1,183 +0,0 @@
-/**
- * Unit tests for toon.ts
- *
- * TOON (Token-Oriented Object Notation) encoder.
- *
- * @module cli/toon.spec
- */
-
-import { describe, it, expect } from 'bun:test';
-import { encode } from './toon';
-
-// ============================================================================
-// encode - primitives
-// ============================================================================
-
-describe('encode - primitives', () => {
-    it('should encode a string primitive', () => {
-        // Top-level primitive string: no key, just the value
-        expect(encode('hello')).toBe('hello');
-    });
-
-    it('should encode a number primitive', () => {
-        expect(encode(42)).toBe('42');
-    });
-
-    it('should encode a boolean primitive', () => {
-        expect(encode(true)).toBe('true');
-        expect(encode(false)).toBe('false');
-    });
-
-    it('should encode null as "null"', () => {
-        expect(encode(null)).toBe('null');
-    });
-
-    it('should encode undefined as "null"', () => {
-        expect(encode(undefined)).toBe('null');
-    });
-
-    it('should quote strings containing the delimiter (tab)', () => {
-        expect(encode('hello\tworld')).toBe('"hello\\tworld"');
-    });
-
-    it('should quote strings containing a colon (:)', () => {
-        expect(encode('key:value')).toBe('"key:value"');
-    });
-
-    it('should quote strings containing a newline', () => {
-        expect(encode('line1\nline2')).toBe('"line1\\nline2"');
-    });
-
-    it('should quote strings containing a double quote', () => {
-        expect(encode('"quoted"')).toBe('"\\"quoted\\""');
-    });
-});
-
-// ============================================================================
-// encode - flat objects
-// ============================================================================
-
-describe('encode - flat objects', () => {
-    it('should encode a single key-value pair', () => {
-        expect(encode({ name: 'test' })).toBe('name:test');
-    });
-
-    it('should encode multiple key-value pairs with tab delimiter', () => {
-        expect(encode({ name: 'test', count: 42 })).toBe('name:test\tcount:42');
-    });
-
-    it('should use custom delimiter when specified', () => {
-        expect(encode({ a: 1, b: 2 }, { delimiter: '|' })).toBe('a:1|b:2');
-    });
-});
-
-// ============================================================================
-// encode - nested objects (no key folding)
-// ============================================================================
-
-describe('encode - nested objects (no key folding)', () => {
-    it('should encode nested object with inline braces', () => {
-        // "user:{name:Alice\trole:admin}"
-        expect(encode({ user: { name: 'Alice', role: 'admin' } })).toBe('user:{name:Alice\trole:admin}');
-    });
-
-    it('should handle deeply nested objects', () => {
-        // "a:{b:{c:deep}}"
-        expect(encode({ a: { b: { c: 'deep' } } })).toBe('a:{b:{c:deep}}');
-    });
-});
-
-// ============================================================================
-// encode - key folding
-// ============================================================================
-
-describe('encode - key folding', () => {
-    it('should flatten nested object to dotted keys with keyFolding:safe', () => {
-        expect(encode({ user: { name: 'Alice', role: 'admin' } }, { keyFolding: 'safe' }))
-            .toBe('user.name:Alice\tuser.role:admin');
-    });
-
-    it('should handle multi-level nesting with keyFolding:safe', () => {
-        expect(encode(
-            { user: { profile: { name: 'Alice' }, settings: { theme: 'dark' } } },
-            { keyFolding: 'safe' },
-        )).toBe('user.profile.name:Alice\tuser.settings.theme:dark');
-    });
-});
-
-// ============================================================================
-// encode - arrays
-// ============================================================================
-
-describe('encode - arrays', () => {
-    it('should encode uniform array of objects in tabular format', () => {
-        const input = {
-            items: [
-                { id: 1, name: 'Alice' },
-                { id: 2, name: 'Bob' },
-            ],
-        };
-        // items[2]{id\tname}:\n\t1\tAlice\n\t2\tBob
-        const expected = 'items[2]{id\tname}:\n\t1\tAlice\n\t2\tBob';
-        expect(encode(input)).toBe(expected);
-    });
-
-    it('should encode empty array as quoted empty string (serializePrimitive path)', () => {
-        // isUniformArray returns false for empty arrays; the else branch in the
-        // object serialiser calls serializePrimitive([], delimiter) which does
-        // quoteString(String([])) → '""'
-        expect(encode({ items: [] })).toBe('items:""');
-    });
-
-    it('should serialize non-uniform array via serializePrimitive (String coercion)', () => {
-        // Non-uniform arrays fall through to the else branch in the object
-        // serialiser → serializePrimitive → quoteString(String([...]))
-        const input = {
-            items: [
-                { id: 1, name: 'Alice' },
-                { id: 2 }, // missing 'name' key — non-uniform
-            ],
-        };
-        expect(encode(input)).toBe('items:"[object Object],[object Object]"');
-    });
-
-    it('should serialize primitive array via serializePrimitive (comma-joined string)', () => {
-        // Primitive arrays also go through serializePrimitive → quoteString(String([...]))
-        expect(encode({ tags: [
-            'a',
-            'b',
-            'c',
-        ] })).toBe('tags:"a,b,c"');
-    });
-
-    it('should serialize array of mixed types via serializePrimitive', () => {
-        expect(encode({ mixed: [
-            1,
-            'two',
-            true,
-        ] })).toBe('mixed:"1,two,true"');
-    });
-});
-
-// ============================================================================
-// encode - edge cases
-// ============================================================================
-
-describe('encode - edge cases', () => {
-    it('should encode an empty object', () => {
-        expect(encode({})).toBe('');
-    });
-
-    it('should handle object with array of uniform items alongside other fields', () => {
-        const input = {
-            title: 'Report',
-            rows: [
-                { id: 1, val: 'a' },
-                { id: 2, val: 'b' },
-            ],
-        };
-        // "title:Report\trows[2]{id\tval}:\n\t1\ta\n\t2\tb"
-        const expected = 'title:Report\trows[2]{id\tval}:\n\t1\ta\n\t2\tb';
-        expect(encode(input)).toBe(expected);
-    });
-});

package/src/toon.ts

@@ -1,462 +0,0 @@
-/**
- * TOON (Token-Oriented Object Notation) encoder.
- *
- * A compact serialization format optimized for LLM context consumption,
- * achieving approximately 40% token reduction compared to JSON while
- * improving parsing accuracy from 70% to 74% in LLM benchmarks.
- *
- * ## Key Features
- *
- * - **Tab delimiters** - Uses tabs between key-value pairs instead of JSON's
- *   commas and braces, maximizing token efficiency
- * - **Key folding** - Collapses nested object paths to dotted notation
- *   (e.g., `user.profile.name` instead of nested objects)
- * - **Tabular arrays** - Uniform object arrays use a compact header+rows format,
- *   dramatically reducing repetition
- *
- * ## Format Specification
- *
- * | JSON Construct | TOON Equivalent |
- * |----------------|-----------------|
- * | `{"key": "value"}` | `key:value` |
- * | `{"a": 1, "b": 2}` | `a:1\tb:2` |
- * | Nested object | `parent.child:value` (with key folding) |
- * | Uniform array | `items[N]{col1\tcol2}:\n\tval1\tval2` |
- *
- * ## Token Efficiency
- *
- * TOON reduces token count through:
- * 1. Eliminating JSON syntax tokens (`{`, `}`, `[`, `]`, `,`, `"`)
- * 2. Using single-character delimiters (tabs, colons)
- * 3. Deduplicating keys in tabular array format
- *
- * Typical savings:
- * - Simple objects: ~30% reduction
- * - Nested objects with key folding: ~40% reduction
- * - Arrays of uniform objects: ~50% reduction
- *
- * @module cli/toon
- *
- * @example Basic object encoding
- * ```ts
- * import { encode } from './toon';
- *
- * encode({ name: 'test', count: 42 });
- * // Output: "name:test\tcount:42"
- * ```
- *
- * @example Key folding for nested objects
- * ```ts
- * encode(
- *   { user: { name: 'Alice', role: 'admin' } },
- *   { keyFolding: 'safe' }
- * );
- * // Output: "user.name:Alice\tuser.role:admin"
- * //
- * // Without key folding:
- * // "user:{name:Alice\trole:admin}"
- * ```
- *
- * @example Tabular arrays (automatic for uniform object arrays)
- * ```ts
- * encode({
- *   items: [
- *     { id: 1, name: 'Widget' },
- *     { id: 2, name: 'Gadget' },
- *   ]
- * });
- * // Output:
- * // "items[2]{id\tname}:
- * //  \t1\tWidget
- * //  \t2\tGadget"
- * //
- * // Equivalent JSON (32 tokens):
- * // {"items":[{"id":1,"name":"Widget"},{"id":2,"name":"Gadget"}]}
- * // TOON version (18 tokens): ~44% reduction
- * ```
- *
- * @see {@link ToonOptions} for configuration options
- * @see {@link encode} for the main encoding function
- */
-
-/**
- * Configuration options for TOON encoding.
- *
- * @example Using custom delimiter
- * ```ts
- * encode({ a: 1, b: 2 }, { delimiter: '|' });
- * // Output: "a:1|b:2"
- * ```
- *
- * @example Enabling key folding
- * ```ts
- * encode({ config: { debug: true } }, { keyFolding: 'safe' });
- * // Output: "config.debug:true"
- * ```
- */
-export interface ToonOptions {
-    /**
-     * Character used to separate key-value pairs.
-     *
-     * The tab character (`\t`) is used by default as it provides optimal
-     * token efficiency in most LLM tokenizers (single token per delimiter).
-     *
-     * @default '\t'
-     */
-    delimiter?: string;
-
-    /**
-     * Controls how nested objects are serialized.
-     *
-     * - `'none'` - Nested objects are serialized inline with braces:
-     *   `parent:{child:value}`
-     * - `'safe'` - Nested paths are collapsed to dotted notation:
-     *   `parent.child:value`
-     *
-     * Key folding (`'safe'`) typically produces more compact output and is
-     * easier for LLMs to parse, but may cause key collisions if object
-     * keys contain dots.
-     *
-     * @default 'none'
-     */
-    keyFolding?: 'safe' | 'none';
-}
-
-/**
- * Default encoding options.
- *
- * - `delimiter`: Tab character for maximum token efficiency
- * - `keyFolding`: Disabled by default for safety (avoids key collisions)
- */
-const DEFAULT_OPTIONS: Required<ToonOptions> = {
-    delimiter: '\t',
-    keyFolding: 'none',
-};
-
-/**
- * Checks if a string needs quoting (contains delimiter, newline, or special chars).
- *
- * Values containing these characters must be quoted to preserve TOON format integrity:
- * - The delimiter character (default: tab)
- * - Newlines (`\n`, `\r`)
- * - Double quotes (`"`)
- * - Colons (`:`) - used as key-value separator
- *
- * @param value - The string value to check
- * @param delimiter - The current delimiter character
- * @returns `true` if the value requires quoting
- */
-const needsQuoting = (value: string, delimiter: string): boolean => {
-    return (
-        value.includes(delimiter) ||
-        value.includes('\n') ||
-        value.includes('\r') ||
-        value.includes('"') ||
-        value.includes(':')
-    );
-};
-
-/**
- * Quotes a string value for TOON output.
- *
- * Uses JSON string escaping rules to ensure special characters are
- * properly escaped in the output.
- *
- * @param value - The string to quote
- * @returns JSON-escaped quoted string
- */
-const quoteString = (value: string): string => {
-    return JSON.stringify(value);
-};
-
-/**
- * Serializes a primitive value to its TOON string representation.
- *
- * Handles the following types:
- * - `null`/`undefined` → `"null"`
- * - `string` → quoted if contains special chars, otherwise raw
- * - `number`/`boolean` → string representation
- * - Other types → JSON-stringified
- *
- * @param value - The primitive value to serialize
- * @param delimiter - The delimiter used for quoting decisions
- * @returns TOON string representation of the value
- */
-const serializePrimitive = (value: unknown, delimiter: string): string => {
-    if (value === null) return 'null';
-    if (value === undefined) return 'null';
-    if (typeof value === 'string') {
-        return needsQuoting(value, delimiter) ? quoteString(value) : value;
-    }
-    if (typeof value === 'number' || typeof value === 'boolean') {
-        return String(value);
-    }
-    return quoteString(String(value));
-};
-
-/**
- * Checks if an array contains uniform objects (all same keys).
- *
- * Uniform arrays qualify for the more compact tabular format where
- * column headers are specified once and rows contain only values.
- *
- * An array is uniform if:
- * 1. It contains at least one element
- * 2. All elements are non-null objects (not arrays)
- * 3. All objects have exactly the same keys
- *
- * @param arr - The array to check
- * @returns Type predicate indicating if array contains uniform objects
- */
-const isUniformArray = (arr: unknown[]): arr is Record<string, unknown>[] => {
-    if (arr.length === 0) return false;
-    if (!arr.every((item) => typeof item === 'object' && item !== null && !Array.isArray(item))) {
-        return false;
-    }
-    const first = arr[0] as Record<string, unknown>;
-    const firstKeys = Object.keys(first).sort().join(',');
-    return arr.every(
-        (item) =>
-            Object.keys(item as object)
-                .sort()
-                .join(',') === firstKeys,
-    );
-};
-
-/**
- * Serializes a uniform array in tabular format.
- *
- * Tabular format provides significant token savings for arrays of objects
- * with identical structure by specifying column headers once.
- *
- * Format: `key[count]{col1\tcol2}:\n\tval1\tval2\n\tval3\tval4`
- *
- * @param arr - Array of uniform objects to serialize
- * @param key - The property name for this array
- * @param delimiter - Character used between values
- * @returns Tabular TOON representation
- *
- * @example
- * ```ts
- * serializeTabularArray(
- *   [{ id: 1, name: 'a' }, { id: 2, name: 'b' }],
- *   'items',
- *   '\t'
- * );
- * // Returns:
- * // "items[2]{id\tname}:
- * //  \t1\ta
- * //  \t2\tb"
- * ```
- */
-const serializeTabularArray = (
-    arr: Record<string, unknown>[],
-    key: string,
-    delimiter: string,
-): string => {
-    if (arr.length === 0) return `${key}:[]`;
-
-    const first = arr[0] as Record<string, unknown>;
-    const headers = Object.keys(first);
-    const headerLine = `${key}[${arr.length}]{${headers.join(delimiter)}}:`;
-
-    const rows = arr.map((item) => {
-        const values = headers.map((h) => serializePrimitive(item[h], delimiter));
-        return delimiter + values.join(delimiter);
-    });
-
-    return [
-        headerLine, ...rows,
-    ].join('\n');
-};
-
-/**
- * Flattens an object with key folding (dotted notation).
- *
- * Recursively traverses nested objects and creates a flat object with
- * dotted key paths. Arrays are preserved as-is (not flattened).
- *
- * @param obj - The object to flatten
- * @param prefix - Current key prefix for recursion (empty string at root)
- * @returns Flattened object with dotted keys
- *
- * @example
- * ```ts
- * flattenObject({ user: { name: 'Alice', settings: { theme: 'dark' } } });
- * // Returns: { 'user.name': 'Alice', 'user.settings.theme': 'dark' }
- * ```
- */
-const flattenObject = (
-    obj: Record<string, unknown>,
-    prefix: string = '',
-): Record<string, unknown> => {
-    const result: Record<string, unknown> = {};
-
-    for (const [
-        key, value,
-    ] of Object.entries(obj)) {
-        const fullKey = prefix ? `${prefix}.${key}` : key;
-
-        if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
-            Object.assign(result, flattenObject(value as Record<string, unknown>, fullKey));
-        }
-        else {
-            result[fullKey] = value;
-        }
-    }
-
-    return result;
-};
-
-/**
- * Serializes a value to TOON format (internal recursive function).
- *
- * Handles all value types and applies the appropriate serialization strategy:
- * - Primitives: Direct serialization with optional quoting
- * - Arrays: Tabular format for uniform objects, JSON for mixed/primitive arrays
- * - Objects: Key folding or inline braces based on options
- *
- * @param value - The value to serialize
- * @param options - Encoding options (fully resolved with defaults)
- * @param key - Optional key name when serializing as an object property
- * @returns TOON-formatted string
- */
-const serializeValue = (value: unknown, options: Required<ToonOptions>, key?: string): string => {
-    const { delimiter, keyFolding } = options;
-
-    // Handle primitives
-    if (value === null || value === undefined) {
-        return key ? `${key}:null` : 'null';
-    }
-
-    if (typeof value !== 'object') {
-        const serialized = serializePrimitive(value, delimiter);
-        return key ? `${key}:${serialized}` : serialized;
-    }
-
-    // Handle arrays
-    if (Array.isArray(value)) {
-        if (isUniformArray(value) && key) {
-            return serializeTabularArray(value, key, delimiter);
-        }
-        // Non-uniform arrays: serialize as JSON array
-        const serialized = JSON.stringify(value);
-        return key ? `${key}:${serialized}` : serialized;
-    }
-
-    // Handle objects
-    let obj = value as Record<string, unknown>;
-    if (keyFolding === 'safe') {
-        obj = flattenObject(obj);
-    }
-
-    const entries = Object.entries(obj);
-    const parts: string[] = [];
-
-    for (const [
-        k, v,
-    ] of entries) {
-        if (Array.isArray(v) && isUniformArray(v)) {
-            parts.push(serializeTabularArray(v, k, delimiter));
-        }
-        else if (
-            typeof v === 'object' &&
-            v !== null &&
-            !Array.isArray(v) &&
-            keyFolding !== 'safe'
-        ) {
-            // Nested object without key folding - serialize inline
-            parts.push(`${k}:{${serializeValue(v, options)}}`);
-        }
-        else {
-            const serialized = serializePrimitive(v, delimiter);
-            parts.push(`${k}:${serialized}`);
-        }
-    }
-
-    const result = parts.join(delimiter);
-    return key ? `${key}:{${result}}` : result;
-};
-
-/**
- * Encodes a value to TOON format.
- *
- * TOON (Token-Oriented Object Notation) is a compact serialization format
- * designed to minimize token consumption in LLM contexts while remaining
- * human-readable and easily parseable by language models.
- *
- * @param value - The value to encode (must be JSON-serializable)
- * @param options - Encoding options for customizing output format
- * @returns TOON-encoded string representation
- *
- * @example Basic object encoding
- * ```ts
- * const data = { name: 'test', count: 42 };
- * encode(data);
- * // Returns: "name:test\tcount:42"
- * //
- * // Equivalent JSON: {"name":"test","count":42}
- * // Token savings: ~35%
- * ```
- *
- * @example Nested objects with key folding
- * ```ts
- * const nested = {
- *   user: {
- *     profile: { name: 'Alice', email: 'alice@example.com' },
- *     settings: { theme: 'dark' }
- *   }
- * };
- *
- * // Without key folding (default):
- * encode(nested);
- * // Returns: "user:{profile:{name:Alice\temail:alice@example.com}\tsettings:{theme:dark}}"
- *
- * // With key folding:
- * encode(nested, { keyFolding: 'safe' });
- * // Returns: "user.profile.name:Alice\tuser.profile.email:alice@example.com\tuser.settings.theme:dark"
- * ```
- *
- * @example Tabular arrays (automatic for uniform object arrays)
- * ```ts
- * const users = {
- *   users: [
- *     { id: 1, name: 'Alice', role: 'admin' },
- *     { id: 2, name: 'Bob', role: 'user' },
- *     { id: 3, name: 'Charlie', role: 'user' },
- *   ]
- * };
- *
- * encode(users);
- * // Returns:
- * // "users[3]{id\tname\trole}:
- * //  \t1\tAlice\tadmin
- * //  \t2\tBob\tuser
- * //  \t3\tCharlie\tuser"
- * //
- * // Equivalent JSON (87 chars, ~25 tokens):
- * // {"users":[{"id":1,"name":"Alice","role":"admin"},{"id":2,"name":"Bob","role":"user"},{"id":3,"name":"Charlie","role":"user"}]}
- * //
- * // TOON (67 chars, ~15 tokens): ~40% token reduction
- * ```
- *
- * @example Mixed content
- * ```ts
- * encode({
- *   title: 'Report',
- *   metadata: { version: 1 },
- *   tags: ['urgent', 'review']  // Non-uniform arrays use JSON format
- * }, { keyFolding: 'safe' });
- * // Returns: "title:Report\tmetadata.version:1\ttags:[\"urgent\",\"review\"]"
- * ```
- *
- * @see {@link ToonOptions} for available configuration options
- */
-export const encode = (value: unknown, options?: ToonOptions): string => {
-    const mergedOptions: Required<ToonOptions> = {
-        ...DEFAULT_OPTIONS,
-        ...options,
-    };
-
-    return serializeValue(value, mergedOptions);
-};