@jtml/core 0.1.0

package/dist/cli.js

@@ -0,0 +1,684 @@
+#!/usr/bin/env node
+"use strict";
+
+// src/cli.ts
+var import_fs = require("fs");
+
+// src/core/types.ts
+var JTMLError = class extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+    this.name = "JTMLError";
+  }
+};
+
+// src/core/schema.ts
+var SchemaManager = class {
+  constructor() {
+    this.schemas = /* @__PURE__ */ new Map();
+  }
+  /**
+   * Register a schema for reuse
+   */
+  register(schema) {
+    this.schemas.set(schema.id, schema);
+  }
+  /**
+   * Get a registered schema
+   */
+  get(id) {
+    return this.schemas.get(id);
+  }
+  /**
+   * Check if schema exists
+   */
+  has(id) {
+    return this.schemas.has(id);
+  }
+  /**
+   * Clear all schemas
+   */
+  clear() {
+    this.schemas.clear();
+  }
+  /**
+   * Export all schemas
+   */
+  export() {
+    return Array.from(this.schemas.values());
+  }
+  /**
+   * Import schemas
+   */
+  import(schemas) {
+    schemas.forEach((schema) => this.register(schema));
+  }
+};
+function inferType(value) {
+  if (value === null || value === void 0) {
+    return { type: "n" };
+  }
+  if (typeof value === "boolean") {
+    return { type: "b" };
+  }
+  if (typeof value === "number") {
+    return Number.isInteger(value) ? { type: "i" } : { type: "f" };
+  }
+  if (typeof value === "string") {
+    if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(value)) {
+      return { type: "t" };
+    }
+    return { type: "s" };
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) {
+      return { type: "a" };
+    }
+    const firstItemType = inferType(value[0]);
+    return { type: "a", arrayOf: firstItemType.type };
+  }
+  if (typeof value === "object") {
+    return { type: "o" };
+  }
+  throw new JTMLError(`Cannot infer type for value: ${value}`, "TYPE_INFERENCE_ERROR");
+}
+function inferSchema(data, schemaId) {
+  if (!Array.isArray(data) && typeof data !== "object") {
+    throw new JTMLError("Schema inference requires array or object data", "INVALID_DATA");
+  }
+  const fields = [];
+  const sample = Array.isArray(data) ? data[0] : data;
+  if (!sample || typeof sample !== "object") {
+    throw new JTMLError("Cannot infer schema from empty or non-object data", "INVALID_DATA");
+  }
+  for (const [key, value] of Object.entries(sample)) {
+    const typeInfo = inferType(value);
+    let optional = false;
+    if (Array.isArray(data)) {
+      optional = data.some(
+        (item) => item[key] === null || item[key] === void 0
+      );
+    }
+    fields.push({
+      name: key,
+      typeInfo: { ...typeInfo, optional }
+    });
+  }
+  return {
+    id: schemaId,
+    fields,
+    version: "1.0"
+  };
+}
+function serializeSchema(schema) {
+  const fieldDefs = schema.fields.map((field) => {
+    let def = `${field.name}:${field.typeInfo.type}`;
+    if (field.typeInfo.arrayOf) {
+      def += `[]`;
+    }
+    if (field.typeInfo.enumValues) {
+      def += `[${field.typeInfo.enumValues.join(",")}]`;
+    }
+    if (field.typeInfo.refSchema) {
+      def += `[${field.typeInfo.refSchema}]`;
+    }
+    if (field.typeInfo.optional) {
+      def += "?";
+    }
+    return def;
+  }).join(" ");
+  return `@schema ${schema.id}
+${fieldDefs}`;
+}
+function parseSchema(schemaStr) {
+  const lines = schemaStr.trim().split("\n");
+  const headerMatch = lines[0].match(/^@schema\s+(\S+)/);
+  if (!headerMatch) {
+    throw new JTMLError("Invalid schema format", "SCHEMA_PARSE_ERROR");
+  }
+  const schemaId = headerMatch[1];
+  if (lines.length < 2 || !lines[1]) {
+    throw new JTMLError("Schema is missing field definitions", "SCHEMA_PARSE_ERROR");
+  }
+  const fieldLine = lines[1];
+  const fields = [];
+  const fieldDefs = fieldLine.split(/\s+/);
+  for (const fieldDef of fieldDefs) {
+    const match = fieldDef.match(/^(\w+):([ifsbtnoae]+)(\[\])?(\[([^\]]+)\])?(\?)?$/);
+    if (!match) {
+      throw new JTMLError(`Invalid field definition: ${fieldDef}`, "SCHEMA_PARSE_ERROR");
+    }
+    const [, name, type, isArray, , enumOrRef, isOptional] = match;
+    const typeInfo = {
+      type,
+      optional: !!isOptional
+    };
+    if (isArray) {
+      typeInfo.arrayOf = type;
+    }
+    if (enumOrRef) {
+      if (type === "e") {
+        typeInfo.enumValues = enumOrRef.split(",");
+      } else if (type === "ref") {
+        typeInfo.refSchema = enumOrRef;
+      }
+    }
+    fields.push({ name, typeInfo });
+  }
+  return {
+    id: schemaId,
+    fields,
+    version: "1.0"
+  };
+}
+var schemaManager = new SchemaManager();
+
+// src/core/encoder.ts
+var JTMLEncoder = class {
+  /**
+   * Encode JSON data to JTML format
+   */
+  encode(data, options = {}) {
+    const {
+      schemaId = "default",
+      schemaRef,
+      autoInferTypes = true,
+      includeSchema = true
+    } = options;
+    let schema;
+    if (schemaRef) {
+      schema = schemaManager.get(schemaRef);
+      if (!schema) {
+        throw new JTMLError(`Schema not found: ${schemaRef}`, "SCHEMA_NOT_FOUND");
+      }
+      return this.encodeWithSchema(data, schema, false);
+    }
+    if (autoInferTypes) {
+      schema = inferSchema(data, schemaId);
+      schemaManager.register(schema);
+    }
+    if (schema) {
+      return this.encodeWithSchema(data, schema, includeSchema);
+    }
+    return this.encodeSimple(data);
+  }
+  /**
+   * Encode with explicit schema
+   */
+  encodeWithSchema(data, schema, includeSchema) {
+    const parts = [];
+    if (includeSchema) {
+      parts.push(serializeSchema(schema));
+      parts.push("");
+      parts.push("@data");
+    } else {
+      parts.push(`@ref ${schema.id}`);
+      parts.push("@data");
+    }
+    if (Array.isArray(data)) {
+      parts.push("@array");
+      for (const item of data) {
+        parts.push(this.encodeRow(item, schema));
+      }
+    } else if (typeof data === "object" && data !== null) {
+      parts.push(this.encodeRow(data, schema));
+    }
+    return parts.join("\n");
+  }
+  /**
+   * Encode a single row according to schema
+   */
+  encodeRow(item, schema) {
+    if (typeof item !== "object" || item === null) {
+      throw new JTMLError("Cannot encode non-object item", "INVALID_DATA");
+    }
+    const values = [];
+    const obj = item;
+    for (const field of schema.fields) {
+      const value = obj[field.name];
+      values.push(this.encodeValue(value));
+    }
+    return values.join("|");
+  }
+  /**
+   * Encode a single value
+   */
+  encodeValue(value) {
+    if (value === null || value === void 0) {
+      return "";
+    }
+    if (typeof value === "boolean") {
+      return value ? "1" : "0";
+    }
+    if (typeof value === "number") {
+      return String(value);
+    }
+    if (typeof value === "string") {
+      return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, "\\n");
+    }
+    if (Array.isArray(value)) {
+      return `[${value.map((v) => this.encodeValue(v)).join(",")}]`;
+    }
+    if (typeof value === "object") {
+      return `{${Object.entries(value).map(([k, v]) => `${k}:${this.encodeValue(v)}`).join(",")}}`;
+    }
+    return String(value);
+  }
+  /**
+   * Simple encoding without schema (fallback)
+   */
+  encodeSimple(data) {
+    if (Array.isArray(data)) {
+      return data.map((item) => JSON.stringify(item)).join("\n");
+    }
+    return JSON.stringify(data);
+  }
+  /**
+   * Encode with metadata
+   */
+  encodeWithMetadata(data, metadata, options = {}) {
+    const dataEncoded = this.encode(data, options);
+    const metaParts = ["", "@meta"];
+    for (const [key, value] of Object.entries(metadata)) {
+      metaParts.push(`${key}:${this.encodeValue(value)}`);
+    }
+    return dataEncoded + "\n" + metaParts.join("\n");
+  }
+};
+var encoder = new JTMLEncoder();
+function encode(data, options) {
+  return encoder.encode(data, options);
+}
+
+// src/core/decoder.ts
+var JTMLDecoder = class {
+  /**
+   * Decode JTML format to JSON
+   */
+  decode(jtml, options = {}) {
+    const { schemaCache, strict = true } = options;
+    if (schemaCache) {
+      schemaCache.forEach((schema2, _id) => {
+        schemaManager.register(schema2);
+      });
+    }
+    const lines = jtml.trim().split("\n");
+    let schema;
+    let dataStartIndex = 0;
+    let metadata = {};
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trim();
+      if (line.startsWith("@schema")) {
+        if (i + 1 >= lines.length) {
+          throw new JTMLError("Incomplete schema definition: missing field definitions", "SCHEMA_PARSE_ERROR");
+        }
+        const schemaLines = [line, lines[i + 1]];
+        schema = parseSchema(schemaLines.join("\n"));
+        schemaManager.register(schema);
+        i++;
+      } else if (line.startsWith("@ref")) {
+        const schemaId = line.split(/\s+/)[1];
+        if (!schemaId) {
+          throw new JTMLError("Missing schema ID in @ref directive", "SCHEMA_PARSE_ERROR");
+        }
+        schema = schemaManager.get(schemaId);
+        if (!schema) {
+          throw new JTMLError(`Schema not found: ${schemaId}`, "SCHEMA_NOT_FOUND");
+        }
+      } else if (line === "@data") {
+        dataStartIndex = i + 1;
+        break;
+      }
+    }
+    if (!schema && strict) {
+      throw new JTMLError("No schema found in JTML data", "SCHEMA_REQUIRED");
+    }
+    let metaStartIndex = -1;
+    for (let i = dataStartIndex; i < lines.length; i++) {
+      if (lines[i].trim() === "@meta") {
+        metaStartIndex = i + 1;
+        break;
+      }
+    }
+    const dataEndIndex = metaStartIndex > 0 ? metaStartIndex - 1 : lines.length;
+    const rawDataLines = lines.slice(dataStartIndex, dataEndIndex).filter((l) => l.trim());
+    const isArrayEncoding = rawDataLines[0]?.trim() === "@array";
+    const dataLines = isArrayEncoding ? rawDataLines.slice(1) : rawDataLines;
+    const results = [];
+    if (schema) {
+      for (const line of dataLines) {
+        if (line.trim()) {
+          results.push(this.decodeRow(line, schema));
+        }
+      }
+    } else {
+      for (const line of dataLines) {
+        if (line.trim()) {
+          results.push(JSON.parse(line));
+        }
+      }
+    }
+    if (metaStartIndex > 0) {
+      for (let i = metaStartIndex; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line && !line.startsWith("@")) {
+          const colonIdx = line.indexOf(":");
+          if (colonIdx === -1) continue;
+          const key = line.slice(0, colonIdx);
+          const value = line.slice(colonIdx + 1);
+          if (key === "__proto__" || key === "constructor" || key === "prototype") continue;
+          metadata[key] = this.decodeValue(value);
+        }
+      }
+    }
+    if (!isArrayEncoding && results.length === 1 && Object.keys(metadata).length === 0) {
+      return results[0];
+    }
+    if (Object.keys(metadata).length > 0) {
+      return {
+        data: results.length === 1 ? results[0] : results,
+        metadata
+      };
+    }
+    return results;
+  }
+  /**
+   * Decode a single row according to schema
+   */
+  decodeRow(line, schema) {
+    const values = this.splitRow(line);
+    const result = {};
+    if (values.length !== schema.fields.length) {
+      throw new JTMLError(
+        `Row has ${values.length} values but schema expects ${schema.fields.length}`,
+        "SCHEMA_MISMATCH"
+      );
+    }
+    for (let i = 0; i < schema.fields.length; i++) {
+      const field = schema.fields[i];
+      const rawValue = values[i];
+      if (rawValue === "" || rawValue === null) {
+        result[field.name] = null;
+      } else {
+        result[field.name] = this.decodeValue(rawValue, field.typeInfo.type);
+      }
+    }
+    return result;
+  }
+  /**
+   * Split row by pipe delimiter, handling escaped pipes
+   */
+  splitRow(line) {
+    const parts = [];
+    let current = "";
+    let escaped = false;
+    for (let i = 0; i < line.length; i++) {
+      const char = line[i];
+      if (escaped) {
+        if (char === "n") {
+          current += "\n";
+        } else if (char === "\\") {
+          current += "\\";
+        } else if (char === "|") {
+          current += "|";
+        } else {
+          current += char;
+        }
+        escaped = false;
+      } else if (char === "\\") {
+        escaped = true;
+      } else if (char === "|") {
+        parts.push(current);
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+    parts.push(current);
+    return parts;
+  }
+  /**
+   * Decode a single value
+   */
+  decodeValue(value, type) {
+    if (value === void 0 || value === "" || value === null) {
+      return null;
+    }
+    if (type === "b") {
+      return value === "1" || value === "true";
+    }
+    if (type === "i" || type === "f") {
+      const num = type === "i" ? parseInt(value, 10) : parseFloat(value);
+      if (!isFinite(num)) {
+        throw new JTMLError(`Invalid numeric value: ${value}`, "INVALID_VALUE");
+      }
+      return num;
+    }
+    if (!type && /^-?\d+(\.\d+)?$/.test(value)) {
+      return value.includes(".") ? parseFloat(value) : parseInt(value, 10);
+    }
+    if (value.startsWith("[") && value.endsWith("]")) {
+      const inner = value.slice(1, -1);
+      if (!inner) return [];
+      return inner.split(",").map((v) => this.decodeValue(v.trim(), type));
+    }
+    if (value.startsWith("{") && value.endsWith("}")) {
+      const inner = value.slice(1, -1);
+      const obj = /* @__PURE__ */ Object.create(null);
+      if (!inner) return obj;
+      const pairs = inner.split(",");
+      for (const pair of pairs) {
+        const colonIdx = pair.indexOf(":");
+        if (colonIdx === -1) continue;
+        const k = pair.slice(0, colonIdx).trim();
+        const v = pair.slice(colonIdx + 1).trim();
+        if (k === "__proto__" || k === "constructor" || k === "prototype") continue;
+        obj[k] = this.decodeValue(v);
+      }
+      return obj;
+    }
+    return value;
+  }
+};
+var decoder = new JTMLDecoder();
+function decode(jtml, options) {
+  return decoder.decode(jtml, options);
+}
+
+// src/utils/tokenizer.ts
+function estimateTokens(text, tokenizer = "claude") {
+  const normalized = text.trim().replace(/\s+/g, " ");
+  const charsPerToken = {
+    gpt: 4,
+    claude: 3.8,
+    llama: 4.2
+  };
+  const ratio = charsPerToken[tokenizer];
+  let estimate = normalized.length / ratio;
+  const structuralChars = (normalized.match(/[{}[\]":,]/g) || []).length;
+  estimate += structuralChars * 0.3;
+  const numbers = (normalized.match(/\d+/g) || []).length;
+  estimate += numbers * 0.2;
+  return Math.ceil(estimate);
+}
+function compareTokens(jsonText, jtmlText, tokenizer = "claude") {
+  const jsonTokens = estimateTokens(jsonText, tokenizer);
+  const jtmlTokens = estimateTokens(jtmlText, tokenizer);
+  const savings = jsonTokens - jtmlTokens;
+  const savingsPercent = jsonTokens > 0 ? savings / jsonTokens * 100 : 0;
+  return {
+    jsonTokens,
+    jtmlTokens,
+    savings,
+    savingsPercent
+  };
+}
+function formatTokenStats(stats) {
+  return `
+Token Comparison:
+  JSON:    ${stats.jsonTokens} tokens
+  JTML:    ${stats.jtmlTokens} tokens
+  Savings: ${stats.savings} tokens (${stats.savingsPercent.toFixed(2)}%)
+  `.trim();
+}
+
+// src/cli.ts
+var args = process.argv.slice(2);
+function printUsage() {
+  console.log(`
+JTML - JSON Token-Minimized Language
+
+Usage:
+  jtml encode <input.json> [output.jtml]   Convert JSON to JTML
+  jtml decode <input.jtml> [output.json]   Convert JTML to JSON
+  jtml compare <input.json>                Compare token efficiency
+  jtml validate <input.jtml>               Validate JTML format
+  jtml schema <input.json>                 Generate schema only
+
+Options:
+  -h, --help                               Show this help message
+  -v, --version                            Show version
+  --schema-id <id>                         Set schema ID
+  --no-schema                              Encode without schema
+  --tokenizer <gpt|claude|llama>           Set tokenizer for comparison
+
+Examples:
+  jtml encode api-response.json            # Output to stdout
+  jtml encode data.json output.jtml        # Save to file
+  jtml decode data.jtml                    # Output to stdout
+  jtml compare large-api.json              # Show token savings
+  `);
+}
+function getVersion() {
+  try {
+    const pkg = JSON.parse((0, import_fs.readFileSync)("./package.json", "utf-8"));
+    return pkg.version;
+  } catch {
+    return "unknown";
+  }
+}
+function main() {
+  if (args.length === 0 || args.includes("-h") || args.includes("--help")) {
+    printUsage();
+    process.exit(0);
+  }
+  if (args.includes("-v") || args.includes("--version")) {
+    console.log(`jtml v${getVersion()}`);
+    process.exit(0);
+  }
+  const command = args[0];
+  const inputFile = args[1];
+  const outputFile = args[2];
+  const schemaIdIndex = args.indexOf("--schema-id");
+  const schemaId = schemaIdIndex >= 0 ? args[schemaIdIndex + 1] : void 0;
+  const noSchema = args.includes("--no-schema");
+  const tokenizerIndex = args.indexOf("--tokenizer");
+  const tokenizer = tokenizerIndex >= 0 ? args[tokenizerIndex + 1] : "claude";
+  try {
+    switch (command) {
+      case "encode": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jsonText = (0, import_fs.readFileSync)(inputFile, "utf-8");
+        const data = JSON.parse(jsonText);
+        const jtml = encode(data, {
+          schemaId,
+          includeSchema: !noSchema,
+          autoInferTypes: true
+        });
+        if (outputFile) {
+          (0, import_fs.writeFileSync)(outputFile, jtml, "utf-8");
+          console.log(`\u2713 Encoded to ${outputFile}`);
+          const stats = compareTokens(jsonText, jtml, tokenizer);
+          console.log(formatTokenStats(stats));
+        } else {
+          console.log(jtml);
+        }
+        break;
+      }
+      case "decode": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jtml = (0, import_fs.readFileSync)(inputFile, "utf-8");
+        const data = decode(jtml);
+        const jsonText = JSON.stringify(data, null, 2);
+        if (outputFile) {
+          (0, import_fs.writeFileSync)(outputFile, jsonText, "utf-8");
+          console.log(`\u2713 Decoded to ${outputFile}`);
+        } else {
+          console.log(jsonText);
+        }
+        break;
+      }
+      case "compare": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jsonText = (0, import_fs.readFileSync)(inputFile, "utf-8");
+        const data = JSON.parse(jsonText);
+        const jtml = encode(data);
+        const stats = compareTokens(jsonText, jtml, tokenizer);
+        console.log("\n=== Token Efficiency Comparison ===\n");
+        console.log(formatTokenStats(stats));
+        console.log(`
+Tokenizer: ${tokenizer}`);
+        console.log(`JSON size: ${jsonText.length} chars`);
+        console.log(`JTML size: ${jtml.length} chars`);
+        console.log(`Compression: ${((jsonText.length - jtml.length) / jsonText.length * 100).toFixed(2)}%`);
+        break;
+      }
+      case "validate": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jtml = (0, import_fs.readFileSync)(inputFile, "utf-8");
+        try {
+          decode(jtml);
+          console.log("\u2713 Valid JTML format");
+        } catch (error) {
+          console.error("\u2717 Invalid JTML format");
+          if (error instanceof Error) {
+            console.error(`  ${error.message}`);
+          }
+          process.exit(1);
+        }
+        break;
+      }
+      case "schema": {
+        if (!inputFile) {
+          console.error("Error: Input file required");
+          printUsage();
+          process.exit(1);
+        }
+        const jsonText = (0, import_fs.readFileSync)(inputFile, "utf-8");
+        const data = JSON.parse(jsonText);
+        const jtml = encode(data, {
+          schemaId: schemaId || "generated",
+          includeSchema: true
+        });
+        const schemaOnly = jtml.split("@data")[0].trim();
+        console.log(schemaOnly);
+        break;
+      }
+      default:
+        console.error(`Error: Unknown command '${command}'`);
+        printUsage();
+        process.exit(1);
+    }
+  } catch (error) {
+    console.error("Error:", error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+}
+main();