@jtml/core 0.1.0

package/dist/index.js

@@ -0,0 +1,726 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  JTMLDecoder: () => JTMLDecoder,
+  JTMLEncoder: () => JTMLEncoder,
+  JTMLError: () => JTMLError,
+  SchemaManager: () => SchemaManager,
+  TYPE_MAP: () => TYPE_MAP,
+  analyzeTokens: () => analyzeTokens,
+  calculateEfficiency: () => calculateEfficiency,
+  compareTokens: () => compareTokens,
+  convertJsonString: () => convertJsonString,
+  decode: () => decode,
+  decoder: () => decoder,
+  default: () => index_default,
+  encode: () => encode,
+  encodeBatch: () => encodeBatch,
+  encoder: () => encoder,
+  estimateCostSavings: () => estimateCostSavings,
+  estimateTokens: () => estimateTokens,
+  formatTokenStats: () => formatTokenStats,
+  inferSchema: () => inferSchema,
+  inferType: () => inferType,
+  jsonToJtml: () => jsonToJtml,
+  jtmlToJson: () => jtmlToJson,
+  parseSchema: () => parseSchema,
+  roundTrip: () => roundTrip,
+  schemaManager: () => schemaManager,
+  serializeSchema: () => serializeSchema,
+  validateAgainstSchema: () => validateAgainstSchema
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/core/types.ts
+var TYPE_MAP = {
+  i: "integer",
+  f: "float",
+  s: "string",
+  b: "boolean",
+  t: "timestamp",
+  n: "null",
+  o: "object",
+  a: "array",
+  e: "enum",
+  ref: "reference"
+};
+var JTMLError = class extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+    this.name = "JTMLError";
+  }
+};
+
+// src/core/schema.ts
+var SchemaManager = class {
+  constructor() {
+    this.schemas = /* @__PURE__ */ new Map();
+  }
+  /**
+   * Register a schema for reuse
+   */
+  register(schema) {
+    this.schemas.set(schema.id, schema);
+  }
+  /**
+   * Get a registered schema
+   */
+  get(id) {
+    return this.schemas.get(id);
+  }
+  /**
+   * Check if schema exists
+   */
+  has(id) {
+    return this.schemas.has(id);
+  }
+  /**
+   * Clear all schemas
+   */
+  clear() {
+    this.schemas.clear();
+  }
+  /**
+   * Export all schemas
+   */
+  export() {
+    return Array.from(this.schemas.values());
+  }
+  /**
+   * Import schemas
+   */
+  import(schemas) {
+    schemas.forEach((schema) => this.register(schema));
+  }
+};
+function inferType(value) {
+  if (value === null || value === void 0) {
+    return { type: "n" };
+  }
+  if (typeof value === "boolean") {
+    return { type: "b" };
+  }
+  if (typeof value === "number") {
+    return Number.isInteger(value) ? { type: "i" } : { type: "f" };
+  }
+  if (typeof value === "string") {
+    if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(value)) {
+      return { type: "t" };
+    }
+    return { type: "s" };
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) {
+      return { type: "a" };
+    }
+    const firstItemType = inferType(value[0]);
+    return { type: "a", arrayOf: firstItemType.type };
+  }
+  if (typeof value === "object") {
+    return { type: "o" };
+  }
+  throw new JTMLError(`Cannot infer type for value: ${value}`, "TYPE_INFERENCE_ERROR");
+}
+function inferSchema(data, schemaId) {
+  if (!Array.isArray(data) && typeof data !== "object") {
+    throw new JTMLError("Schema inference requires array or object data", "INVALID_DATA");
+  }
+  const fields = [];
+  const sample = Array.isArray(data) ? data[0] : data;
+  if (!sample || typeof sample !== "object") {
+    throw new JTMLError("Cannot infer schema from empty or non-object data", "INVALID_DATA");
+  }
+  for (const [key, value] of Object.entries(sample)) {
+    const typeInfo = inferType(value);
+    let optional = false;
+    if (Array.isArray(data)) {
+      optional = data.some(
+        (item) => item[key] === null || item[key] === void 0
+      );
+    }
+    fields.push({
+      name: key,
+      typeInfo: { ...typeInfo, optional }
+    });
+  }
+  return {
+    id: schemaId,
+    fields,
+    version: "1.0"
+  };
+}
+function serializeSchema(schema) {
+  const fieldDefs = schema.fields.map((field) => {
+    let def = `${field.name}:${field.typeInfo.type}`;
+    if (field.typeInfo.arrayOf) {
+      def += `[]`;
+    }
+    if (field.typeInfo.enumValues) {
+      def += `[${field.typeInfo.enumValues.join(",")}]`;
+    }
+    if (field.typeInfo.refSchema) {
+      def += `[${field.typeInfo.refSchema}]`;
+    }
+    if (field.typeInfo.optional) {
+      def += "?";
+    }
+    return def;
+  }).join(" ");
+  return `@schema ${schema.id}
+${fieldDefs}`;
+}
+function parseSchema(schemaStr) {
+  const lines = schemaStr.trim().split("\n");
+  const headerMatch = lines[0].match(/^@schema\s+(\S+)/);
+  if (!headerMatch) {
+    throw new JTMLError("Invalid schema format", "SCHEMA_PARSE_ERROR");
+  }
+  const schemaId = headerMatch[1];
+  if (lines.length < 2 || !lines[1]) {
+    throw new JTMLError("Schema is missing field definitions", "SCHEMA_PARSE_ERROR");
+  }
+  const fieldLine = lines[1];
+  const fields = [];
+  const fieldDefs = fieldLine.split(/\s+/);
+  for (const fieldDef of fieldDefs) {
+    const match = fieldDef.match(/^(\w+):([ifsbtnoae]+)(\[\])?(\[([^\]]+)\])?(\?)?$/);
+    if (!match) {
+      throw new JTMLError(`Invalid field definition: ${fieldDef}`, "SCHEMA_PARSE_ERROR");
+    }
+    const [, name, type, isArray, , enumOrRef, isOptional] = match;
+    const typeInfo = {
+      type,
+      optional: !!isOptional
+    };
+    if (isArray) {
+      typeInfo.arrayOf = type;
+    }
+    if (enumOrRef) {
+      if (type === "e") {
+        typeInfo.enumValues = enumOrRef.split(",");
+      } else if (type === "ref") {
+        typeInfo.refSchema = enumOrRef;
+      }
+    }
+    fields.push({ name, typeInfo });
+  }
+  return {
+    id: schemaId,
+    fields,
+    version: "1.0"
+  };
+}
+function validateAgainstSchema(data, schema) {
+  if (Array.isArray(data)) {
+    return data.every((item) => validateItem(item, schema));
+  }
+  return validateItem(data, schema);
+}
+function validateItem(item, schema) {
+  if (typeof item !== "object" || item === null) {
+    return false;
+  }
+  for (const field of schema.fields) {
+    const value = item[field.name];
+    if (value === null || value === void 0) {
+      if (!field.typeInfo.optional) {
+        return false;
+      }
+      continue;
+    }
+    const actualType = inferType(value);
+    if (actualType.type !== field.typeInfo.type) {
+      return false;
+    }
+  }
+  return true;
+}
+var schemaManager = new SchemaManager();
+
+// src/core/encoder.ts
+var JTMLEncoder = class {
+  /**
+   * Encode JSON data to JTML format
+   */
+  encode(data, options = {}) {
+    const {
+      schemaId = "default",
+      schemaRef,
+      autoInferTypes = true,
+      includeSchema = true
+    } = options;
+    let schema;
+    if (schemaRef) {
+      schema = schemaManager.get(schemaRef);
+      if (!schema) {
+        throw new JTMLError(`Schema not found: ${schemaRef}`, "SCHEMA_NOT_FOUND");
+      }
+      return this.encodeWithSchema(data, schema, false);
+    }
+    if (autoInferTypes) {
+      schema = inferSchema(data, schemaId);
+      schemaManager.register(schema);
+    }
+    if (schema) {
+      return this.encodeWithSchema(data, schema, includeSchema);
+    }
+    return this.encodeSimple(data);
+  }
+  /**
+   * Encode with explicit schema
+   */
+  encodeWithSchema(data, schema, includeSchema) {
+    const parts = [];
+    if (includeSchema) {
+      parts.push(serializeSchema(schema));
+      parts.push("");
+      parts.push("@data");
+    } else {
+      parts.push(`@ref ${schema.id}`);
+      parts.push("@data");
+    }
+    if (Array.isArray(data)) {
+      parts.push("@array");
+      for (const item of data) {
+        parts.push(this.encodeRow(item, schema));
+      }
+    } else if (typeof data === "object" && data !== null) {
+      parts.push(this.encodeRow(data, schema));
+    }
+    return parts.join("\n");
+  }
+  /**
+   * Encode a single row according to schema
+   */
+  encodeRow(item, schema) {
+    if (typeof item !== "object" || item === null) {
+      throw new JTMLError("Cannot encode non-object item", "INVALID_DATA");
+    }
+    const values = [];
+    const obj = item;
+    for (const field of schema.fields) {
+      const value = obj[field.name];
+      values.push(this.encodeValue(value));
+    }
+    return values.join("|");
+  }
+  /**
+   * Encode a single value
+   */
+  encodeValue(value) {
+    if (value === null || value === void 0) {
+      return "";
+    }
+    if (typeof value === "boolean") {
+      return value ? "1" : "0";
+    }
+    if (typeof value === "number") {
+      return String(value);
+    }
+    if (typeof value === "string") {
+      return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, "\\n");
+    }
+    if (Array.isArray(value)) {
+      return `[${value.map((v) => this.encodeValue(v)).join(",")}]`;
+    }
+    if (typeof value === "object") {
+      return `{${Object.entries(value).map(([k, v]) => `${k}:${this.encodeValue(v)}`).join(",")}}`;
+    }
+    return String(value);
+  }
+  /**
+   * Simple encoding without schema (fallback)
+   */
+  encodeSimple(data) {
+    if (Array.isArray(data)) {
+      return data.map((item) => JSON.stringify(item)).join("\n");
+    }
+    return JSON.stringify(data);
+  }
+  /**
+   * Encode with metadata
+   */
+  encodeWithMetadata(data, metadata, options = {}) {
+    const dataEncoded = this.encode(data, options);
+    const metaParts = ["", "@meta"];
+    for (const [key, value] of Object.entries(metadata)) {
+      metaParts.push(`${key}:${this.encodeValue(value)}`);
+    }
+    return dataEncoded + "\n" + metaParts.join("\n");
+  }
+};
+var encoder = new JTMLEncoder();
+function encode(data, options) {
+  return encoder.encode(data, options);
+}
+function encodeBatch(datasets, schemaId = "batch") {
+  if (datasets.length === 0) {
+    throw new JTMLError("Cannot encode empty batch", "INVALID_DATA");
+  }
+  const schema = inferSchema(datasets[0], schemaId);
+  schemaManager.register(schema);
+  const parts = [];
+  parts.push(serializeSchema(schema));
+  parts.push("");
+  datasets.forEach((data, index) => {
+    parts.push(`@batch ${index}`);
+    const encoded = encoder.encode(data, {
+      schemaRef: schemaId,
+      includeSchema: false
+    });
+    parts.push(encoded);
+    parts.push("");
+  });
+  return parts.join("\n");
+}
+
+// src/core/decoder.ts
+var JTMLDecoder = class {
+  /**
+   * Decode JTML format to JSON
+   */
+  decode(jtml, options = {}) {
+    const { schemaCache, strict = true } = options;
+    if (schemaCache) {
+      schemaCache.forEach((schema2, _id) => {
+        schemaManager.register(schema2);
+      });
+    }
+    const lines = jtml.trim().split("\n");
+    let schema;
+    let dataStartIndex = 0;
+    let metadata = {};
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trim();
+      if (line.startsWith("@schema")) {
+        if (i + 1 >= lines.length) {
+          throw new JTMLError("Incomplete schema definition: missing field definitions", "SCHEMA_PARSE_ERROR");
+        }
+        const schemaLines = [line, lines[i + 1]];
+        schema = parseSchema(schemaLines.join("\n"));
+        schemaManager.register(schema);
+        i++;
+      } else if (line.startsWith("@ref")) {
+        const schemaId = line.split(/\s+/)[1];
+        if (!schemaId) {
+          throw new JTMLError("Missing schema ID in @ref directive", "SCHEMA_PARSE_ERROR");
+        }
+        schema = schemaManager.get(schemaId);
+        if (!schema) {
+          throw new JTMLError(`Schema not found: ${schemaId}`, "SCHEMA_NOT_FOUND");
+        }
+      } else if (line === "@data") {
+        dataStartIndex = i + 1;
+        break;
+      }
+    }
+    if (!schema && strict) {
+      throw new JTMLError("No schema found in JTML data", "SCHEMA_REQUIRED");
+    }
+    let metaStartIndex = -1;
+    for (let i = dataStartIndex; i < lines.length; i++) {
+      if (lines[i].trim() === "@meta") {
+        metaStartIndex = i + 1;
+        break;
+      }
+    }
+    const dataEndIndex = metaStartIndex > 0 ? metaStartIndex - 1 : lines.length;
+    const rawDataLines = lines.slice(dataStartIndex, dataEndIndex).filter((l) => l.trim());
+    const isArrayEncoding = rawDataLines[0]?.trim() === "@array";
+    const dataLines = isArrayEncoding ? rawDataLines.slice(1) : rawDataLines;
+    const results = [];
+    if (schema) {
+      for (const line of dataLines) {
+        if (line.trim()) {
+          results.push(this.decodeRow(line, schema));
+        }
+      }
+    } else {
+      for (const line of dataLines) {
+        if (line.trim()) {
+          results.push(JSON.parse(line));
+        }
+      }
+    }
+    if (metaStartIndex > 0) {
+      for (let i = metaStartIndex; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line && !line.startsWith("@")) {
+          const colonIdx = line.indexOf(":");
+          if (colonIdx === -1) continue;
+          const key = line.slice(0, colonIdx);
+          const value = line.slice(colonIdx + 1);
+          if (key === "__proto__" || key === "constructor" || key === "prototype") continue;
+          metadata[key] = this.decodeValue(value);
+        }
+      }
+    }
+    if (!isArrayEncoding && results.length === 1 && Object.keys(metadata).length === 0) {
+      return results[0];
+    }
+    if (Object.keys(metadata).length > 0) {
+      return {
+        data: results.length === 1 ? results[0] : results,
+        metadata
+      };
+    }
+    return results;
+  }
+  /**
+   * Decode a single row according to schema
+   */
+  decodeRow(line, schema) {
+    const values = this.splitRow(line);
+    const result = {};
+    if (values.length !== schema.fields.length) {
+      throw new JTMLError(
+        `Row has ${values.length} values but schema expects ${schema.fields.length}`,
+        "SCHEMA_MISMATCH"
+      );
+    }
+    for (let i = 0; i < schema.fields.length; i++) {
+      const field = schema.fields[i];
+      const rawValue = values[i];
+      if (rawValue === "" || rawValue === null) {
+        result[field.name] = null;
+      } else {
+        result[field.name] = this.decodeValue(rawValue, field.typeInfo.type);
+      }
+    }
+    return result;
+  }
+  /**
+   * Split row by pipe delimiter, handling escaped pipes
+   */
+  splitRow(line) {
+    const parts = [];
+    let current = "";
+    let escaped = false;
+    for (let i = 0; i < line.length; i++) {
+      const char = line[i];
+      if (escaped) {
+        if (char === "n") {
+          current += "\n";
+        } else if (char === "\\") {
+          current += "\\";
+        } else if (char === "|") {
+          current += "|";
+        } else {
+          current += char;
+        }
+        escaped = false;
+      } else if (char === "\\") {
+        escaped = true;
+      } else if (char === "|") {
+        parts.push(current);
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+    parts.push(current);
+    return parts;
+  }
+  /**
+   * Decode a single value
+   */
+  decodeValue(value, type) {
+    if (value === void 0 || value === "" || value === null) {
+      return null;
+    }
+    if (type === "b") {
+      return value === "1" || value === "true";
+    }
+    if (type === "i" || type === "f") {
+      const num = type === "i" ? parseInt(value, 10) : parseFloat(value);
+      if (!isFinite(num)) {
+        throw new JTMLError(`Invalid numeric value: ${value}`, "INVALID_VALUE");
+      }
+      return num;
+    }
+    if (!type && /^-?\d+(\.\d+)?$/.test(value)) {
+      return value.includes(".") ? parseFloat(value) : parseInt(value, 10);
+    }
+    if (value.startsWith("[") && value.endsWith("]")) {
+      const inner = value.slice(1, -1);
+      if (!inner) return [];
+      return inner.split(",").map((v) => this.decodeValue(v.trim(), type));
+    }
+    if (value.startsWith("{") && value.endsWith("}")) {
+      const inner = value.slice(1, -1);
+      const obj = /* @__PURE__ */ Object.create(null);
+      if (!inner) return obj;
+      const pairs = inner.split(",");
+      for (const pair of pairs) {
+        const colonIdx = pair.indexOf(":");
+        if (colonIdx === -1) continue;
+        const k = pair.slice(0, colonIdx).trim();
+        const v = pair.slice(colonIdx + 1).trim();
+        if (k === "__proto__" || k === "constructor" || k === "prototype") continue;
+        obj[k] = this.decodeValue(v);
+      }
+      return obj;
+    }
+    return value;
+  }
+};
+var decoder = new JTMLDecoder();
+function decode(jtml, options) {
+  return decoder.decode(jtml, options);
+}
+
+// src/utils/tokenizer.ts
+function estimateTokens(text, tokenizer = "claude") {
+  const normalized = text.trim().replace(/\s+/g, " ");
+  const charsPerToken = {
+    gpt: 4,
+    claude: 3.8,
+    llama: 4.2
+  };
+  const ratio = charsPerToken[tokenizer];
+  let estimate = normalized.length / ratio;
+  const structuralChars = (normalized.match(/[{}[\]":,]/g) || []).length;
+  estimate += structuralChars * 0.3;
+  const numbers = (normalized.match(/\d+/g) || []).length;
+  estimate += numbers * 0.2;
+  return Math.ceil(estimate);
+}
+function compareTokens(jsonText, jtmlText, tokenizer = "claude") {
+  const jsonTokens = estimateTokens(jsonText, tokenizer);
+  const jtmlTokens = estimateTokens(jtmlText, tokenizer);
+  const savings = jsonTokens - jtmlTokens;
+  const savingsPercent = jsonTokens > 0 ? savings / jsonTokens * 100 : 0;
+  return {
+    jsonTokens,
+    jtmlTokens,
+    savings,
+    savingsPercent
+  };
+}
+function calculateEfficiency(jsonText, jtmlText) {
+  const jsonLen = jsonText.length;
+  const jtmlLen = jtmlText.length;
+  return (jsonLen - jtmlLen) / jsonLen * 100;
+}
+function analyzeTokens(jsonText, jtmlText, tokenizer = "claude") {
+  const jsonStructural = (jsonText.match(/[{}[\]":,]/g) || []).length;
+  const jsonKeys = (jsonText.match(/"(\w+)":/g) || []).length;
+  const jtmlSchema = jtmlText.split("@data")[0] || "";
+  const jtmlData = jtmlText.split("@data")[1] || "";
+  const jtmlDelimiters = (jtmlText.match(/[|:]/g) || []).length;
+  return {
+    json: {
+      total: estimateTokens(jsonText, tokenizer),
+      structural: Math.ceil(jsonStructural * 0.8),
+      keys: Math.ceil(jsonKeys * 1.5),
+      // Keys with quotes
+      values: estimateTokens(jsonText, tokenizer) - Math.ceil(jsonStructural * 0.8) - Math.ceil(jsonKeys * 1.5)
+    },
+    jtml: {
+      total: estimateTokens(jtmlText, tokenizer),
+      schema: estimateTokens(jtmlSchema, tokenizer),
+      data: estimateTokens(jtmlData, tokenizer),
+      delimiters: Math.ceil(jtmlDelimiters * 0.5)
+    },
+    comparison: compareTokens(jsonText, jtmlText, tokenizer)
+  };
+}
+function formatTokenStats(stats) {
+  return `
+Token Comparison:
+  JSON:    ${stats.jsonTokens} tokens
+  JTML:    ${stats.jtmlTokens} tokens
+  Savings: ${stats.savings} tokens (${stats.savingsPercent.toFixed(2)}%)
+  `.trim();
+}
+function estimateCostSavings(stats, pricePerMillion = 3) {
+  const costSavedPer1M = stats.savings / 1e6 * pricePerMillion;
+  const costSavedPer1K = stats.savings / 1e3 * (pricePerMillion / 1e3);
+  return {
+    tokensSaved: stats.savings,
+    costSavedPer1M,
+    costSavedPer1K
+  };
+}
+
+// src/index.ts
+function jsonToJtml(json, schemaId) {
+  return encode(json, { schemaId, autoInferTypes: true });
+}
+function jtmlToJson(jtml) {
+  return decode(jtml);
+}
+function convertJsonString(jsonStr, schemaId) {
+  const data = JSON.parse(jsonStr);
+  return jsonToJtml(data, schemaId);
+}
+function roundTrip(data) {
+  const jtml = encode(data);
+  const recovered = decode(jtml);
+  return {
+    success: JSON.stringify(data) === JSON.stringify(recovered),
+    original: data,
+    recovered,
+    jtml
+  };
+}
+var index_default = {
+  encode,
+  decode,
+  jsonToJtml,
+  jtmlToJson,
+  convertJsonString,
+  roundTrip,
+  schemaManager,
+  estimateTokens,
+  compareTokens
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  JTMLDecoder,
+  JTMLEncoder,
+  JTMLError,
+  SchemaManager,
+  TYPE_MAP,
+  analyzeTokens,
+  calculateEfficiency,
+  compareTokens,
+  convertJsonString,
+  decode,
+  decoder,
+  encode,
+  encodeBatch,
+  encoder,
+  estimateCostSavings,
+  estimateTokens,
+  formatTokenStats,
+  inferSchema,
+  inferType,
+  jsonToJtml,
+  jtmlToJson,
+  parseSchema,
+  roundTrip,
+  schemaManager,
+  serializeSchema,
+  validateAgainstSchema
+});