@creationix/jot 0.0.1 → 0.1.0

package/dist/jot.js

@@ -0,0 +1,442 @@
+const RESERVED = new Set(["true", "false", "null"])
+const UNSAFE = [":", ",", "{", "}", "[", "]", '"', ";", "\\"]
+const WS_RE = /\s/
+const KEY_TERM_RE = /[:\,{}\[\];]|\s/
+function needsQuotes(s, extra = []) {
+  const chars = [...UNSAFE, ...extra]
+  return (
+    s === "" ||
+    s.trim() !== s ||
+    RESERVED.has(s) ||
+    !Number.isNaN(Number(s)) ||
+    chars.some((c) => s.includes(c)) ||
+    [...s].some((c) => c.charCodeAt(0) < 32)
+  )
+}
+const quote = (s) => (needsQuotes(s) ? JSON.stringify(s) : s)
+const quoteKey = (s) => (needsQuotes(s, ["."]) ? JSON.stringify(s) : s)
+function getFoldPath(value) {
+  const path = []
+  let current = value
+  while (current !== null && typeof current === "object" && !Array.isArray(current)) {
+    const keys = Object.keys(current)
+    if (keys.length !== 1 || keys[0].includes(".")) {
+      break
+    }
+    path.push(keys[0])
+    current = current[keys[0]]
+  }
+  return path.length > 0 ? { path, leaf: current } : null
+}
+function groupBySchema(arr) {
+  const groups = []
+  for (const obj of arr) {
+    const keys = Object.keys(obj)
+    const last = groups.at(-1)
+    if (last && last.keys.join(",") === keys.join(",")) {
+      last.objects.push(obj)
+    } else {
+      groups.push({ keys, objects: [obj] })
+    }
+  }
+  return groups
+}
+let opts = {}
+let depth = 0
+const ind = () => (opts.pretty ? (opts.indent ?? "  ").repeat(depth) : "")
+function stringifyValue(value, atLineStart = false) {
+  if (value === null) {
+    return "null"
+  }
+  if (typeof value === "boolean") {
+    return String(value)
+  }
+  if (typeof value === "number") {
+    return String(value)
+  }
+  if (typeof value === "string") {
+    return quote(value)
+  }
+  if (Array.isArray(value)) {
+    return stringifyArray(value)
+  }
+  if (typeof value === "object") {
+    return stringifyObject(value, atLineStart)
+  }
+  return String(value)
+}
+function stringifyArray(arr) {
+  const isTable = arr.length >= 2 && arr.every((i) => i !== null && typeof i === "object" && !Array.isArray(i))
+  if (isTable) {
+    const groups = groupBySchema(arr)
+    if (groups.some((g) => g.objects.length >= 2)) {
+      return stringifyTable(groups)
+    }
+  }
+  if (arr.length === 1) {
+    return `[${stringifyValue(arr[0])}]`
+  }
+  const hasComplex = arr.some((i) => i !== null && typeof i === "object")
+  if (opts.pretty && arr.length > 0 && hasComplex) {
+    depth++
+    const items = arr.map((i) => `${ind()}${stringifyValue(i, true)}`)
+    depth--
+    return `[\n${items.join(",\n")}\n${ind()}]`
+  }
+  const sep = opts.pretty ? ", " : ","
+  const items = arr.map((v) => stringifyValue(v)).join(sep)
+  return opts.pretty ? `[ ${items} ]` : `[${items}]`
+}
+function stringifyTable(groups) {
+  const sep = opts.pretty ? ", " : ","
+  if (opts.pretty) {
+    depth++
+    const schemaInd = ind()
+    depth++
+    const dataInd = ind()
+    const rows = []
+    for (const { keys, objects } of groups) {
+      rows.push(`${schemaInd}:${keys.map((k) => quoteKey(k)).join(sep)}`)
+      for (const obj of objects) rows.push(`${dataInd}${keys.map((k) => stringifyValue(obj[k])).join(sep)}`)
+    }
+    depth -= 2
+    return `{{\n${rows.join("\n")}\n${ind()}}}`
+  }
+  const parts = []
+  for (const { keys, objects } of groups) {
+    parts.push(`:${keys.map((k) => quoteKey(k)).join(sep)}`)
+    for (const obj of objects) {
+      parts.push(keys.map((k) => stringifyValue(obj[k])).join(sep))
+    }
+  }
+  return `{{${parts.join(";")}}}`
+}
+function stringifyObject(obj, atLineStart = false) {
+  const keys = Object.keys(obj)
+  const pair = (k, pretty) => {
+    const val = obj[k]
+    if (!needsQuotes(k, ["."]) && val !== null && typeof val === "object" && !Array.isArray(val)) {
+      const fold = getFoldPath(val)
+      if (fold) {
+        const foldedKey = `${k}.${fold.path.join(".")}`
+        return pretty ? `${foldedKey}: ${stringifyValue(fold.leaf)}` : `${foldedKey}:${stringifyValue(fold.leaf)}`
+      }
+    }
+    const qk = quoteKey(k)
+    return pretty ? `${qk}: ${stringifyValue(val)}` : `${qk}:${stringifyValue(val)}`
+  }
+  if (opts.pretty && keys.length > 1) {
+    depth++
+    const rawPairs = keys.map((k) => pair(k, true))
+    const lastMulti = rawPairs.at(-1)?.endsWith("}") || rawPairs.at(-1)?.endsWith("]")
+    const compact = atLineStart && !lastMulti
+    const pairs = rawPairs.map((p, i) => (i === 0 && compact ? p : `${ind()}${p}`))
+    depth--
+    return compact ? `{ ${pairs.join(",\n")} }` : `{\n${pairs.join(",\n")}\n${ind()}}`
+  }
+  if (opts.pretty && keys.length === 1) {
+    return `{ ${pair(keys[0], true)} }`
+  }
+  return `{${keys.map((k) => pair(k, false)).join(",")}}`
+}
+export function stringify(data, options = {}) {
+  opts = { pretty: false, indent: "  ", ...options }
+  depth = 0
+  return stringifyValue(data)
+}
+// Parser
+class JotParser {
+  input
+  pos = 0
+  constructor(input) {
+    this.input = input
+  }
+  parse() {
+    this.ws()
+    const result = this.value("")
+    this.ws()
+    if (this.pos < this.input.length) {
+      throw new Error(`Unexpected character at position ${this.pos}: '${this.input[this.pos]}'`)
+    }
+    return result
+  }
+  ws() {
+    while (this.pos < this.input.length && WS_RE.test(this.input[this.pos])) this.pos++
+  }
+  peek = () => this.input[this.pos] || ""
+  value(terminators = "") {
+    this.ws()
+    const ch = this.peek()
+    if (ch === "{") {
+      return this.input[this.pos + 1] === "{" ? this.table() : this.object()
+    }
+    if (ch === "[") {
+      return this.array()
+    }
+    if (ch === '"') {
+      return this.quoted()
+    }
+    return this.atom(terminators)
+  }
+  quoted() {
+    this.pos++
+    let result = ""
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos]
+      if (ch === '"') {
+        this.pos++
+        return result
+      }
+      if (ch === "\\") {
+        this.pos++
+        const esc = this.input[this.pos]
+        const escMap = {
+          '"': '"',
+          "\\": "\\",
+          "/": "/",
+          b: "\b",
+          f: "\f",
+          n: "\n",
+          r: "\r",
+          t: "\t",
+        }
+        if (esc in escMap) {
+          result += escMap[esc]
+        } else if (esc === "u") {
+          result += String.fromCharCode(Number.parseInt(this.input.slice(this.pos + 1, this.pos + 5), 16))
+          this.pos += 4
+        } else {
+          throw new Error(`Invalid escape sequence '\\${esc}'`)
+        }
+      } else {
+        result += ch
+      }
+      this.pos++
+    }
+    throw new Error("Unterminated string")
+  }
+  parseToken(terminators) {
+    const start = this.pos
+    if (terminators === "") {
+      const token = this.input.slice(start).trim()
+      this.pos = this.input.length
+      if (token === "") {
+        throw new Error(`Unexpected end of input at position ${start}`)
+      }
+      return token
+    }
+    while (this.pos < this.input.length && !terminators.includes(this.input[this.pos])) {
+      this.pos++
+    }
+    const token = this.input.slice(start, this.pos).trim()
+    if (token === "") {
+      throw new Error(`Unexpected character at position ${this.pos}: '${this.peek()}'`)
+    }
+    return token
+  }
+  tokenToValue(token) {
+    if (token === "null") {
+      return null
+    }
+    if (token === "true") {
+      return true
+    }
+    if (token === "false") {
+      return false
+    }
+    const num = Number(token)
+    if (!Number.isNaN(num) && token !== "") {
+      return num
+    }
+    return token
+  }
+  atom(terminators) {
+    return this.tokenToValue(this.parseToken(terminators))
+  }
+  array() {
+    this.pos++
+    const result = []
+    this.ws()
+    while (this.peek() !== "]") {
+      if (this.pos >= this.input.length) {
+        throw new Error("Unterminated array")
+      }
+      result.push(this.value(",]"))
+      this.ws()
+      if (this.peek() === ",") {
+        this.pos++
+        this.ws()
+      }
+    }
+    this.pos++
+    return result
+  }
+  table() {
+    this.pos += 2
+    const result = []
+    let schema = []
+    this.ws()
+    while (this.input.slice(this.pos, this.pos + 2) !== "}}") {
+      if (this.pos >= this.input.length) {
+        throw new Error("Unterminated table")
+      }
+      this.ws()
+      if (this.peek() === ":") {
+        this.pos++
+        schema = this.schemaRow()
+      } else {
+        if (schema.length === 0) {
+          throw new Error(`Data row without schema at position ${this.pos}`)
+        }
+        const values = this.dataRow(schema.length)
+        const obj = {}
+        for (let i = 0; i < schema.length; i++) {
+          obj[schema[i]] = values[i]
+        }
+        result.push(obj)
+      }
+      this.ws()
+      if (this.peek() === ";") {
+        this.pos++
+        this.ws()
+      }
+    }
+    this.pos += 2
+    return result
+  }
+  schemaRow() {
+    const cols = []
+    let col = ""
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos]
+      if ((ch === "}" && this.input[this.pos + 1] === "}") || ch === ";" || ch === "\n") {
+        if (col.trim()) {
+          cols.push(col.trim())
+        }
+        break
+      }
+      if (ch === ",") {
+        if (col.trim()) {
+          cols.push(col.trim())
+        }
+        col = ""
+        this.pos++
+        continue
+      }
+      col += ch
+      this.pos++
+    }
+    return cols
+  }
+  dataRow(colCount) {
+    const values = []
+    for (let i = 0; i < colCount; i++) {
+      this.ws()
+      values.push(this.tableValue(i < colCount - 1 ? ",;}\n" : ";}\n"))
+      this.ws()
+      if (this.peek() === ",") {
+        this.pos++
+      }
+    }
+    return values
+  }
+  tableValue(terminators) {
+    this.ws()
+    const ch = this.peek()
+    if (ch === '"') {
+      return this.quoted()
+    }
+    if (ch === "{") {
+      return this.input[this.pos + 1] === "{" ? this.table() : this.object()
+    }
+    if (ch === "[") {
+      return this.array()
+    }
+    const start = this.pos
+    while (this.pos < this.input.length) {
+      const c = this.input[this.pos]
+      if ((c === "}" && this.input[this.pos + 1] === "}") || terminators.includes(c)) {
+        break
+      }
+      this.pos++
+    }
+    const token = this.input.slice(start, this.pos).trim()
+    return token === "" ? null : this.tokenToValue(token)
+  }
+  object() {
+    this.pos++
+    const result = {}
+    this.ws()
+    while (this.peek() !== "}") {
+      if (this.pos >= this.input.length) {
+        throw new Error("Unterminated object")
+      }
+      const { key, quoted } = this.parseKey()
+      this.ws()
+      if (this.peek() !== ":") {
+        throw new Error(`Expected ':' after key '${key}' at position ${this.pos}`)
+      }
+      this.pos++
+      const value = this.value(",}")
+      if (quoted) {
+        result[key] = value
+      } else {
+        this.merge(result, this.unfold(key, value))
+      }
+      this.ws()
+      if (this.peek() === ",") {
+        this.pos++
+        this.ws()
+      }
+    }
+    this.pos++
+    return result
+  }
+  parseKey() {
+    this.ws()
+    if (this.peek() === '"') {
+      return { key: this.quoted(), quoted: true }
+    }
+    const start = this.pos
+    while (this.pos < this.input.length && !KEY_TERM_RE.test(this.input[this.pos])) this.pos++
+    const key = this.input.slice(start, this.pos)
+    if (key === "") {
+      throw new Error(`Expected key at position ${this.pos}`)
+    }
+    return { key, quoted: false }
+  }
+  unfold(keyPath, value) {
+    const parts = keyPath.split(".")
+    const result = {}
+    let current = result
+    for (let i = 0; i < parts.length - 1; i++) {
+      const nested = {}
+      current[parts[i]] = nested
+      current = nested
+    }
+    current[parts.at(-1)] = value
+    return result
+  }
+  merge(target, src) {
+    for (const key of Object.keys(src)) {
+      const tv = target[key]
+      const sv = src[key]
+      if (
+        key in target &&
+        typeof tv === "object" &&
+        tv !== null &&
+        !Array.isArray(tv) &&
+        typeof sv === "object" &&
+        sv !== null &&
+        !Array.isArray(sv)
+      ) {
+        this.merge(tv, sv)
+      } else {
+        target[key] = sv
+      }
+    }
+  }
+}
+export function parse(input) {
+  return new JotParser(input).parse()
+}

package/package.json

@@ -1,10 +1,40 @@
 {
   "name": "@creationix/jot",
-  "version": "0.0.1",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/creationix/jot"
+  },
+  "version": "0.1.0",
+  "description": "LLM and human friendly JSON alternative",
+  "type": "module",
+  "main": "./dist/jot.cjs",
+  "module": "./dist/jot.js",
+  "types": "./dist/jot.d.ts",
+  "exports": {
+    "import": "./dist/jot.js",
+    "require": "./dist/jot.cjs"
+  },
+  "files": [
+    "dist/*",
+    "README.md"
+  ],
+  "scripts": {
+    "build:clean": "rm -rf dist/*",
+    "build:cjs": "tsc --project tsconfig.cjs.json && mv dist/jot.js dist/jot.cjs",
+    "build:esm": "tsc --project tsconfig.json",
+    "build:tidy": "biome format --fix dist && biome lint --fix dist",
+    "build": "npm run build:clean && npm run build:cjs && npm run build:esm && npm run build:tidy",
+    "format": "biome format --fix src && biome lint --fix src",
+    "lint": "biome lint src",
+    "test": "bun test",
+    "prepublish": "npm run format && npm run lint && npm run build && bun samples/measure-tokens.ts"
+  },
   "author": "Tim Caswell <tim@creationix.com>",
   "license": "MIT",
-  "dependencies": {},
   "devDependencies": {
-    "@types/bun": "^1.3.5"
+    "@anthropic-ai/tokenizer": "^0.0.4",
+    "@biomejs/biome": "^1.9.4",
+    "@types/bun": "^1.3.5",
+    "typescript": "^5.7.3"
   }
-}
+}

package/SUMMARY.md

@@ -1,151 +0,0 @@
-# Encoding Format Comparison
-
-Token counts for 18 test documents across three tokenizers. For LLM systems, **tokens matter more than bytes**.
-
-## Recommendation
-
-**Use Jot** for LLM contexts — saves 16-17% tokens vs JSON.
-
-## Token Efficiency
-
-<!-- CHART_START -->
-```mermaid
-xychart-beta
-    title "Token Counts by Format"
-    x-axis ["Jot", "JSON-m", "JSONito", "Jot-P", "D2", "TOON", "YAML", "TOML", "JSON-s", "JSON-p"]
-    y-axis "Tokens" 0 --> 16000
-    line "Qwen" [6525, 7748, 7757, 8239, 8292, 8315, 9543, 10180, 11799, 12656]
-    line "Legacy" [6420, 7377, 7794, 7204, 7582, 7079, 7661, 11204, 10966, 11937]
-    line "Claude" [6747, 8132, 8327, 8500, 7928, 8405, 9456, 11485, 12687, 14403]
-```
-<!-- CHART_END -->
-
-### Compact Formats
-
-For machine-to-machine or LLM contexts where readability isn't required.
-
-<!-- COMPACT_START -->
-| Format                                              | Qwen           | Legacy         | Claude         | Bytes          |
-|-----------------------------------------------------|---------------:|---------------:|---------------:|---------------:|
-| **[Jot](jot/)**                                     |   6,525 (-16%) |   6,420 (-13%) |   6,747 (-17%) |  16,621 (-28%) |
-| [JSON](https://www.json.org/) (mini)                |          7,748 |          7,377 |          8,132 |         23,119 |
-| [JSONito](https://github.com/creationix/jsonito)    |    7,757 (+0%) |    7,794 (+6%) |    8,327 (+2%) |  14,059 (-39%) |
-| [D2](https://github.com/creationix/d2)              |    8,292 (+7%) |    7,582 (+3%) |    7,928 (-3%) |  17,328 (-25%) |
-<!-- COMPACT_END -->
-
-### Pretty-Printed Formats
-
-For human-readable output or when LLMs need to read/write structured data.
-
-<!-- PRETTY_START -->
-| Format                                              | Qwen           | Legacy         | Claude         | Bytes          |
-|-----------------------------------------------------|---------------:|---------------:|---------------:|---------------:|
-| **[Jot](jot/) (pretty)**                            |   8,239 (-35%) |   7,204 (-40%) |   8,500 (-41%) |  23,676 (-41%) |
-| [TOON](toon/)                                       |   8,315 (-34%) |   7,079 (-41%) |   8,405 (-42%) |  22,780 (-43%) |
-| [YAML](https://yaml.org/)                           |   9,543 (-25%) |   7,661 (-36%) |   9,456 (-34%) |  26,757 (-33%) |
-| [TOML](https://toml.io/)                            |  10,180 (-20%) |   11,204 (-6%) |  11,485 (-20%) |  28,930 (-27%) |
-| [JSON](json/smart-json.ts) (smart)                  |   11,799 (-7%) |   10,966 (-8%) |  12,687 (-12%) |  32,657 (-18%) |
-| [JSON](https://www.json.org/) (pretty)              |         12,656 |         11,937 |         14,403 |         39,884 |
-<!-- PRETTY_END -->
-
-## Format Descriptions
-
-### [Jot](jot/)
-
-JSON with three optimizations:
-
-1. **Unquoted strings** — omit quotes unless value contains `: ; , { } [ ] "` or parses as number/boolean/null
-2. **Key folding** — `{a:{b:1}}` → `{a.b:1}` for single-key nested objects
-3. **Tables** — `[{a:1},{a:2}]` → `{{:a;1;2}}` for repeating object schemas
-
-```jot
-{config.host:localhost,users:{{:id,name;1,Alice;2,Bob}}}
-```
-
-It also has a pretty-printed variant that adds indentation and newlines for readability.
-
-```jot
-{
-  config.host: localhost,
-  users: {{
-    :id, name;
-      1, Alice;
-      2, Bob
-  }}
-}
-```
-
-### [TOON](toon/)
-
-YAML-like indentation with optional table syntax and count guards.
-
-```toon
-users[2]{id,name}:
-  1,Alice
-  2,Bob
-```
-
-### [JSONito](https://github.com/creationix/jsonito)
-
-Byte-optimized JSON with string deduplication via preamble dictionary.
-
-```jito
-{name'config'version'5~1.0.0enabled'!a~maxRetries6.timeout'eFw.tags'[a~productionapi'v1']}
-```
-
-### [D2](https://github.com/creationix/d2)
-
-Declarative data format using `=` assignment and shell-like quoting.
-
-## Why Not Byte-Optimized Formats?
-
-Formats like JSONito achieve excellent byte compression (-39%) but:
-
-- Token savings are inconsistent (small docs often cost more than JSON)
-- Deduplication preambles add overhead that doesn't scale down
-- LLMs cannot reliably generate formats requiring state tracking
-
-## LLM Encoding Ability
-
-Tested Qwen3-30b's ability to encode JSON → Jot (3 runs per document, 17 docs):
-
-| Document Type                                | Semantic Accuracy |
-|----------------------------------------------|------------------:|
-| Simple configs (small, metrics, package)     |              100% |
-| Key folding test cases                       |              100% |
-| Table-friendly (users-50)                    |              100% |
-| Text-heavy (chat)                            |              100% |
-| Complex/nested (large, firewall, routes)     |                0% |
-| Irregular schemas (medium, hikes, irregular) |                0% |
-| **Overall**                                  |           **47%** |
-
-Small models struggle with Jot's advanced features on complex documents. For LLM-generated output, consider using simpler Jot (unquoted strings only) or providing FORMAT.md as context.
-
-## Tokenizers
-
-- **Qwen**: Qwen3-Coder-30b via LM Studio API
-- **Legacy**: Anthropic legacy tokenizer (`@anthropic-ai/tokenizer`)
-- **Claude**: Claude API token counting endpoint (Sonnet/Opus/Haiku share tokenizer)
-
-## Test Data
-
-18 documents covering diverse structures:
-
-| Document          | Description                      |
-|-------------------|----------------------------------|
-| small             | Config object (6 fields)         |
-| medium            | User list with metadata          |
-| large             | Kubernetes deployment spec       |
-| hikes             | Tabular records (uniform schema) |
-| chat              | LLM conversation (text-heavy)    |
-| metrics           | Time series (numeric-heavy)      |
-| package           | npm manifest (nested deps)       |
-| github-issue      | Mixed nesting with labels        |
-| irregular         | Event log (varying keys)         |
-| users-50          | 50 user records (table-friendly) |
-| logs              | 50 log entries (semi-uniform)    |
-| firewall          | WAF rules (deeply nested)        |
-| products          | E-commerce catalog (variants)    |
-| routes            | API routing config (large tables)|
-| key-folding-*     | Key folding test cases           |
-| json-counts-cache | Cached token counts              |