@jtml/core 0.1.0

package/dist/cli.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+import {
+  compareTokens,
+  decode,
+  encode,
+  formatTokenStats
+} from "./chunk-SHDXMADE.mjs";
+
+// src/cli.ts
+import { readFileSync, writeFileSync } from "fs";
+var args = process.argv.slice(2);
+function printUsage() {
+  console.log(`
+JTML - JSON Token-Minimized Language
+
+Usage:
+  jtml encode <input.json> [output.jtml]   Convert JSON to JTML
+  jtml decode <input.jtml> [output.json]   Convert JTML to JSON
+  jtml compare <input.json>                Compare token efficiency
+  jtml validate <input.jtml>               Validate JTML format
+  jtml schema <input.json>                 Generate schema only
+
+Options:
+  -h, --help                               Show this help message
+  -v, --version                            Show version
+  --schema-id <id>                         Set schema ID
+  --no-schema                              Encode without schema
+  --tokenizer <gpt|claude|llama>           Set tokenizer for comparison
+
+Examples:
+  jtml encode api-response.json            # Output to stdout
+  jtml encode data.json output.jtml        # Save to file
+  jtml decode data.jtml                    # Output to stdout
+  jtml compare large-api.json              # Show token savings
+  `);
+}
+function getVersion() {
+  try {
+    const pkg = JSON.parse(readFileSync("./package.json", "utf-8"));
+    return pkg.version;
+  } catch {
+    return "unknown";
+  }
+}
+function main() {
+  if (args.length === 0 || args.includes("-h") || args.includes("--help")) {
+    printUsage();
+    process.exit(0);
+  }
+  if (args.includes("-v") || args.includes("--version")) {
+    console.log(`jtml v${getVersion()}`);
+    process.exit(0);
+  }
+  const command = args[0];
+  const inputFile = args[1];
+  const outputFile = args[2];
+  const schemaIdIndex = args.indexOf("--schema-id");
+  const schemaId = schemaIdIndex >= 0 ? args[schemaIdIndex + 1] : void 0;
+  const noSchema = args.includes("--no-schema");
+  const tokenizerIndex = args.indexOf("--tokenizer");
+  const tokenizer = tokenizerIndex >= 0 ? args[tokenizerIndex + 1] : "claude";
+  try {
+    switch (command) {
+      case "encode": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jsonText = readFileSync(inputFile, "utf-8");
+        const data = JSON.parse(jsonText);
+        const jtml = encode(data, {
+          schemaId,
+          includeSchema: !noSchema,
+          autoInferTypes: true
+        });
+        if (outputFile) {
+          writeFileSync(outputFile, jtml, "utf-8");
+          console.log(`\u2713 Encoded to ${outputFile}`);
+          const stats = compareTokens(jsonText, jtml, tokenizer);
+          console.log(formatTokenStats(stats));
+        } else {
+          console.log(jtml);
+        }
+        break;
+      }
+      case "decode": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jtml = readFileSync(inputFile, "utf-8");
+        const data = decode(jtml);
+        const jsonText = JSON.stringify(data, null, 2);
+        if (outputFile) {
+          writeFileSync(outputFile, jsonText, "utf-8");
+          console.log(`\u2713 Decoded to ${outputFile}`);
+        } else {
+          console.log(jsonText);
+        }
+        break;
+      }
+      case "compare": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jsonText = readFileSync(inputFile, "utf-8");
+        const data = JSON.parse(jsonText);
+        const jtml = encode(data);
+        const stats = compareTokens(jsonText, jtml, tokenizer);
+        console.log("\n=== Token Efficiency Comparison ===\n");
+        console.log(formatTokenStats(stats));
+        console.log(`
+Tokenizer: ${tokenizer}`);
+        console.log(`JSON size: ${jsonText.length} chars`);
+        console.log(`JTML size: ${jtml.length} chars`);
+        console.log(`Compression: ${((jsonText.length - jtml.length) / jsonText.length * 100).toFixed(2)}%`);
+        break;
+      }
+      case "validate": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jtml = readFileSync(inputFile, "utf-8");
+        try {
+          decode(jtml);
+          console.log("\u2713 Valid JTML format");
+        } catch (error) {
+          console.error("\u2717 Invalid JTML format");
+          if (error instanceof Error) {
+            console.error(`  ${error.message}`);
+          }
+          process.exit(1);
+        }
+        break;
+      }
+      case "schema": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jsonText = readFileSync(inputFile, "utf-8");
+        const data = JSON.parse(jsonText);
+        const jtml = encode(data, {
+          schemaId: schemaId || "generated",
+          includeSchema: true
+        });
+        const schemaOnly = jtml.split("@data")[0].trim();
+        console.log(schemaOnly);
+        break;
+      }
+      default:
+        console.error(`Error: Unknown command '${command}'`);
+        printUsage();
+        process.exit(1);
+    }
+  } catch (error) {
+    console.error("Error:", error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+}
+main();

package/dist/index.d.mts

@@ -0,0 +1,262 @@
+/**
+ * JTML Type Definitions
+ */
+type JTMLType = 'i' | 'f' | 's' | 'b' | 't' | 'n' | 'o' | 'a' | 'e' | 'ref';
+interface JTMLTypeInfo {
+    type: JTMLType;
+    arrayOf?: JTMLType;
+    enumValues?: string[];
+    refSchema?: string;
+    optional?: boolean;
+}
+interface JTMLField {
+    name: string;
+    typeInfo: JTMLTypeInfo;
+}
+interface JTMLSchema {
+    id: string;
+    fields: JTMLField[];
+    version?: string;
+}
+interface JTMLEncodeOptions {
+    schemaId?: string;
+    schemaRef?: string;
+    autoInferTypes?: boolean;
+    compress?: boolean;
+    includeSchema?: boolean;
+}
+interface JTMLDecodeOptions {
+    schemaCache?: Map<string, JTMLSchema>;
+    strict?: boolean;
+}
+interface JTMLDocument {
+    schema?: JTMLSchema;
+    data: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface TokenStats {
+    jsonTokens: number;
+    jtmlTokens: number;
+    savings: number;
+    savingsPercent: number;
+}
+type JTMLValue = string | number | boolean | null | JTMLValue[] | {
+    [key: string]: JTMLValue;
+};
+/**
+ * Type mapping:
+ * i - integer
+ * f - float
+ * s - string
+ * b - boolean
+ * t - timestamp/datetime
+ * n - null
+ * o - object
+ * a - array
+ * e - enum
+ * ref - reference to another schema
+ */
+declare const TYPE_MAP: Record<JTMLType, string>;
+declare class JTMLError extends Error {
+    code: string;
+    constructor(message: string, code: string);
+}
+
+declare class JTMLEncoder {
+    /**
+     * Encode JSON data to JTML format
+     */
+    encode(data: unknown, options?: JTMLEncodeOptions): string;
+    /**
+     * Encode with explicit schema
+     */
+    private encodeWithSchema;
+    /**
+     * Encode a single row according to schema
+     */
+    private encodeRow;
+    /**
+     * Encode a single value
+     */
+    private encodeValue;
+    /**
+     * Simple encoding without schema (fallback)
+     */
+    private encodeSimple;
+    /**
+     * Encode with metadata
+     */
+    encodeWithMetadata(data: unknown, metadata: Record<string, unknown>, options?: JTMLEncodeOptions): string;
+}
+declare const encoder: JTMLEncoder;
+/**
+ * Convenience function to encode JSON to JTML
+ */
+declare function encode(data: unknown, options?: JTMLEncodeOptions): string;
+/**
+ * Encode multiple datasets with shared schema
+ */
+declare function encodeBatch(datasets: unknown[], schemaId?: string): string;
+
+declare class JTMLDecoder {
+    /**
+     * Decode JTML format to JSON
+     */
+    decode(jtml: string, options?: JTMLDecodeOptions): unknown;
+    /**
+     * Decode a single row according to schema
+     */
+    private decodeRow;
+    /**
+     * Split row by pipe delimiter, handling escaped pipes
+     */
+    private splitRow;
+    /**
+     * Decode a single value
+     */
+    private decodeValue;
+}
+declare const decoder: JTMLDecoder;
+/**
+ * Convenience function to decode JTML to JSON
+ */
+declare function decode(jtml: string, options?: JTMLDecodeOptions): unknown;
+
+/**
+ * Approximate token counting for different tokenizers
+ * Based on heuristics - for exact counts, use actual tokenizer libraries
+ */
+type TokenizerType = 'gpt' | 'claude' | 'llama';
+/**
+ * Estimate token count using simple heuristics
+ * GPT/Claude: ~4 chars per token average
+ * More accurate for estimation than character count
+ */
+declare function estimateTokens(text: string, tokenizer?: TokenizerType): number;
+/**
+ * Compare token efficiency between JSON and JTML
+ */
+declare function compareTokens(jsonText: string, jtmlText: string, tokenizer?: TokenizerType): TokenStats;
+/**
+ * Calculate token efficiency ratio
+ */
+declare function calculateEfficiency(jsonText: string, jtmlText: string): number;
+/**
+ * Detailed token analysis
+ */
+interface DetailedTokenStats {
+    json: {
+        total: number;
+        structural: number;
+        keys: number;
+        values: number;
+    };
+    jtml: {
+        total: number;
+        schema: number;
+        data: number;
+        delimiters: number;
+    };
+    comparison: TokenStats;
+}
+declare function analyzeTokens(jsonText: string, jtmlText: string, tokenizer?: TokenizerType): DetailedTokenStats;
+/**
+ * Format token stats for display
+ */
+declare function formatTokenStats(stats: TokenStats): string;
+/**
+ * Estimate cost savings (based on typical LLM pricing)
+ */
+interface CostSavings {
+    tokensSaved: number;
+    costSavedPer1M: number;
+    costSavedPer1K: number;
+}
+declare function estimateCostSavings(stats: TokenStats, pricePerMillion?: number): CostSavings;
+
+declare class SchemaManager {
+    private schemas;
+    /**
+     * Register a schema for reuse
+     */
+    register(schema: JTMLSchema): void;
+    /**
+     * Get a registered schema
+     */
+    get(id: string): JTMLSchema | undefined;
+    /**
+     * Check if schema exists
+     */
+    has(id: string): boolean;
+    /**
+     * Clear all schemas
+     */
+    clear(): void;
+    /**
+     * Export all schemas
+     */
+    export(): JTMLSchema[];
+    /**
+     * Import schemas
+     */
+    import(schemas: JTMLSchema[]): void;
+}
+/**
+ * Infer JTML type from JavaScript value
+ */
+declare function inferType(value: unknown): JTMLTypeInfo;
+/**
+ * Infer schema from JSON data
+ */
+declare function inferSchema(data: unknown, schemaId: string): JTMLSchema;
+/**
+ * Serialize schema to JTML format
+ */
+declare function serializeSchema(schema: JTMLSchema): string;
+/**
+ * Parse schema from JTML format
+ */
+declare function parseSchema(schemaStr: string): JTMLSchema;
+/**
+ * Validate data against schema
+ */
+declare function validateAgainstSchema(data: unknown, schema: JTMLSchema): boolean;
+declare const schemaManager: SchemaManager;
+
+/**
+ * Quick conversion utilities
+ */
+/**
+ * Convert JSON to JTML (one-liner)
+ */
+declare function jsonToJtml(json: unknown, schemaId?: string): string;
+/**
+ * Convert JTML to JSON (one-liner)
+ */
+declare function jtmlToJson(jtml: string): unknown;
+/**
+ * Convert JSON string to JTML string
+ */
+declare function convertJsonString(jsonStr: string, schemaId?: string): string;
+/**
+ * Round-trip conversion test
+ */
+declare function roundTrip(data: unknown): {
+    success: boolean;
+    original: unknown;
+    recovered: unknown;
+    jtml: string;
+};
+declare const _default: {
+    encode: typeof encode;
+    decode: typeof decode;
+    jsonToJtml: typeof jsonToJtml;
+    jtmlToJson: typeof jtmlToJson;
+    convertJsonString: typeof convertJsonString;
+    roundTrip: typeof roundTrip;
+    schemaManager: SchemaManager;
+    estimateTokens: typeof estimateTokens;
+    compareTokens: typeof compareTokens;
+};
+
+export { type CostSavings, type DetailedTokenStats, type JTMLDecodeOptions, JTMLDecoder, type JTMLDocument, type JTMLEncodeOptions, JTMLEncoder, JTMLError, type JTMLField, type JTMLSchema, type JTMLType, type JTMLTypeInfo, type JTMLValue, SchemaManager, TYPE_MAP, type TokenStats, type TokenizerType, analyzeTokens, calculateEfficiency, compareTokens, convertJsonString, decode, decoder, _default as default, encode, encodeBatch, encoder, estimateCostSavings, estimateTokens, formatTokenStats, inferSchema, inferType, jsonToJtml, jtmlToJson, parseSchema, roundTrip, schemaManager, serializeSchema, validateAgainstSchema };

package/dist/index.d.ts

@@ -0,0 +1,262 @@
+/**
+ * JTML Type Definitions
+ */
+type JTMLType = 'i' | 'f' | 's' | 'b' | 't' | 'n' | 'o' | 'a' | 'e' | 'ref';
+interface JTMLTypeInfo {
+    type: JTMLType;
+    arrayOf?: JTMLType;
+    enumValues?: string[];
+    refSchema?: string;
+    optional?: boolean;
+}
+interface JTMLField {
+    name: string;
+    typeInfo: JTMLTypeInfo;
+}
+interface JTMLSchema {
+    id: string;
+    fields: JTMLField[];
+    version?: string;
+}
+interface JTMLEncodeOptions {
+    schemaId?: string;
+    schemaRef?: string;
+    autoInferTypes?: boolean;
+    compress?: boolean;
+    includeSchema?: boolean;
+}
+interface JTMLDecodeOptions {
+    schemaCache?: Map<string, JTMLSchema>;
+    strict?: boolean;
+}
+interface JTMLDocument {
+    schema?: JTMLSchema;
+    data: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface TokenStats {
+    jsonTokens: number;
+    jtmlTokens: number;
+    savings: number;
+    savingsPercent: number;
+}
+type JTMLValue = string | number | boolean | null | JTMLValue[] | {
+    [key: string]: JTMLValue;
+};
+/**
+ * Type mapping:
+ * i - integer
+ * f - float
+ * s - string
+ * b - boolean
+ * t - timestamp/datetime
+ * n - null
+ * o - object
+ * a - array
+ * e - enum
+ * ref - reference to another schema
+ */
+declare const TYPE_MAP: Record<JTMLType, string>;
+declare class JTMLError extends Error {
+    code: string;
+    constructor(message: string, code: string);
+}
+
+declare class JTMLEncoder {
+    /**
+     * Encode JSON data to JTML format
+     */
+    encode(data: unknown, options?: JTMLEncodeOptions): string;
+    /**
+     * Encode with explicit schema
+     */
+    private encodeWithSchema;
+    /**
+     * Encode a single row according to schema
+     */
+    private encodeRow;
+    /**
+     * Encode a single value
+     */
+    private encodeValue;
+    /**
+     * Simple encoding without schema (fallback)
+     */
+    private encodeSimple;
+    /**
+     * Encode with metadata
+     */
+    encodeWithMetadata(data: unknown, metadata: Record<string, unknown>, options?: JTMLEncodeOptions): string;
+}
+declare const encoder: JTMLEncoder;
+/**
+ * Convenience function to encode JSON to JTML
+ */
+declare function encode(data: unknown, options?: JTMLEncodeOptions): string;
+/**
+ * Encode multiple datasets with shared schema
+ */
+declare function encodeBatch(datasets: unknown[], schemaId?: string): string;
+
+declare class JTMLDecoder {
+    /**
+     * Decode JTML format to JSON
+     */
+    decode(jtml: string, options?: JTMLDecodeOptions): unknown;
+    /**
+     * Decode a single row according to schema
+     */
+    private decodeRow;
+    /**
+     * Split row by pipe delimiter, handling escaped pipes
+     */
+    private splitRow;
+    /**
+     * Decode a single value
+     */
+    private decodeValue;
+}
+declare const decoder: JTMLDecoder;
+/**
+ * Convenience function to decode JTML to JSON
+ */
+declare function decode(jtml: string, options?: JTMLDecodeOptions): unknown;
+
+/**
+ * Approximate token counting for different tokenizers
+ * Based on heuristics - for exact counts, use actual tokenizer libraries
+ */
+type TokenizerType = 'gpt' | 'claude' | 'llama';
+/**
+ * Estimate token count using simple heuristics
+ * GPT/Claude: ~4 chars per token average
+ * More accurate for estimation than character count
+ */
+declare function estimateTokens(text: string, tokenizer?: TokenizerType): number;
+/**
+ * Compare token efficiency between JSON and JTML
+ */
+declare function compareTokens(jsonText: string, jtmlText: string, tokenizer?: TokenizerType): TokenStats;
+/**
+ * Calculate token efficiency ratio
+ */
+declare function calculateEfficiency(jsonText: string, jtmlText: string): number;
+/**
+ * Detailed token analysis
+ */
+interface DetailedTokenStats {
+    json: {
+        total: number;
+        structural: number;
+        keys: number;
+        values: number;
+    };
+    jtml: {
+        total: number;
+        schema: number;
+        data: number;
+        delimiters: number;
+    };
+    comparison: TokenStats;
+}
+declare function analyzeTokens(jsonText: string, jtmlText: string, tokenizer?: TokenizerType): DetailedTokenStats;
+/**
+ * Format token stats for display
+ */
+declare function formatTokenStats(stats: TokenStats): string;
+/**
+ * Estimate cost savings (based on typical LLM pricing)
+ */
+interface CostSavings {
+    tokensSaved: number;
+    costSavedPer1M: number;
+    costSavedPer1K: number;
+}
+declare function estimateCostSavings(stats: TokenStats, pricePerMillion?: number): CostSavings;
+
+declare class SchemaManager {
+    private schemas;
+    /**
+     * Register a schema for reuse
+     */
+    register(schema: JTMLSchema): void;
+    /**
+     * Get a registered schema
+     */
+    get(id: string): JTMLSchema | undefined;
+    /**
+     * Check if schema exists
+     */
+    has(id: string): boolean;
+    /**
+     * Clear all schemas
+     */
+    clear(): void;
+    /**
+     * Export all schemas
+     */
+    export(): JTMLSchema[];
+    /**
+     * Import schemas
+     */
+    import(schemas: JTMLSchema[]): void;
+}
+/**
+ * Infer JTML type from JavaScript value
+ */
+declare function inferType(value: unknown): JTMLTypeInfo;
+/**
+ * Infer schema from JSON data
+ */
+declare function inferSchema(data: unknown, schemaId: string): JTMLSchema;
+/**
+ * Serialize schema to JTML format
+ */
+declare function serializeSchema(schema: JTMLSchema): string;
+/**
+ * Parse schema from JTML format
+ */
+declare function parseSchema(schemaStr: string): JTMLSchema;
+/**
+ * Validate data against schema
+ */
+declare function validateAgainstSchema(data: unknown, schema: JTMLSchema): boolean;
+declare const schemaManager: SchemaManager;
+
+/**
+ * Quick conversion utilities
+ */
+/**
+ * Convert JSON to JTML (one-liner)
+ */
+declare function jsonToJtml(json: unknown, schemaId?: string): string;
+/**
+ * Convert JTML to JSON (one-liner)
+ */
+declare function jtmlToJson(jtml: string): unknown;
+/**
+ * Convert JSON string to JTML string
+ */
+declare function convertJsonString(jsonStr: string, schemaId?: string): string;
+/**
+ * Round-trip conversion test
+ */
+declare function roundTrip(data: unknown): {
+    success: boolean;
+    original: unknown;
+    recovered: unknown;
+    jtml: string;
+};
+declare const _default: {
+    encode: typeof encode;
+    decode: typeof decode;
+    jsonToJtml: typeof jsonToJtml;
+    jtmlToJson: typeof jtmlToJson;
+    convertJsonString: typeof convertJsonString;
+    roundTrip: typeof roundTrip;
+    schemaManager: SchemaManager;
+    estimateTokens: typeof estimateTokens;
+    compareTokens: typeof compareTokens;
+};
+
+export { type CostSavings, type DetailedTokenStats, type JTMLDecodeOptions, JTMLDecoder, type JTMLDocument, type JTMLEncodeOptions, JTMLEncoder, JTMLError, type JTMLField, type JTMLSchema, type JTMLType, type JTMLTypeInfo, type JTMLValue, SchemaManager, TYPE_MAP, type TokenStats, type TokenizerType, analyzeTokens, calculateEfficiency, compareTokens, convertJsonString, decode, decoder, _default as default, encode, encodeBatch, encoder, estimateCostSavings, estimateTokens, formatTokenStats, inferSchema, inferType, jsonToJtml, jtmlToJson, parseSchema, roundTrip, schemaManager, serializeSchema, validateAgainstSchema };