@creationix/jot 0.0.1

package/README.md

@@ -0,0 +1,92 @@
+# Jot Format
+
+Jot is a compact, LLM friendly JSON variant designed to use fewer tokens while remaining easy to read and write.
+
+```jot
+{
+  context: {
+    task: Our favorite hikes together,
+    location: Boulder,
+    season: spring_2025
+  },
+  friends: [ ana, luis, sam ],
+  hikes: {{
+    :id, name, distanceKm, elevationGain, companion, wasSunny
+      1, Blue Lake Trail, 7.5, 320, ana, true
+      2, Ridge Overlook, 9.2, 540, luis, false
+      3, Wildflower Loop, 5.1, 180, sam, true
+  }}
+}
+```
+
+It is JSON with three optimizations:
+
+1. **Unquoted strings** — Strings are only quoted if necessary.
+2. **Key folding** — Single-key nested objects collapse: `{a:{b:1}}` → `{a.b:1}`
+   if normal keys contain dots, keep quotes: `{"a.b":1}`
+3. **Tables** — Object arrays with repeating schemas use `{{:cols;row;row}}` syntax
+
+## Unquoted Strings
+
+The only times that you need to quote a string are:
+
+- It is a valid JSON value (`true`, `false`, `null`, or a number like `42`, `3.14`, `-0.5`, or `1e10`)
+- It contains special characters: `: ; , { } [ ] "` or control characters (newline, tab, etc)
+- It is empty or has leading or trailing whitespace
+- It being used as a key in an object and contains `.` (to distinguish from folded keys)
+
+```json
+{"name":"Alice","city":"New York","count":"42"}
+```
+
+```jot
+{name:Alice,city:New York,count:"42"}
+```
+
+## Key Folding
+
+When a nested object has exactly ONE key, fold it:
+
+```json
+{"server":{"host":"localhost"}}
+```
+
+```jot
+{server.host:localhost}
+```
+
+If normal keys contain dots, keep quotes to avoid confusion:
+
+```json
+{"data.point":{"x":10,"y":20}}
+```
+
+```jot
+{"data.point":{x:10,y:20}}
+```
+
+## Tables
+
+One common shape in data is a table — an array of multiple objects with the same schema.
+
+Object arrays use `{{:schema;row;row;...}}` when schemas repeat. Start with `:` followed by column names:
+
+Don't use tables when there's no schema reuse (each object unique) — regular arrays are more compact.
+
+```json
+[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]
+```
+
+```jot
+{{:id,name;1,Alice;2,Bob}}
+```
+
+To change schema mid-table, add another `:schema;` row:
+
+```json
+[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"},{"x":10,"y":20},{"x":30,"y":40}]
+```
+
+```jot
+{{:id,name;1,Alice;2,Bob;:x,y;10,20;30,40}}
+```

package/SUMMARY.md

@@ -0,0 +1,151 @@
+# Encoding Format Comparison
+
+Token counts for 18 test documents across three tokenizers. For LLM systems, **tokens matter more than bytes**.
+
+## Recommendation
+
+**Use Jot** for LLM contexts — saves 16-17% tokens vs JSON.
+
+## Token Efficiency
+
+<!-- CHART_START -->
+```mermaid
+xychart-beta
+    title "Token Counts by Format"
+    x-axis ["Jot", "JSON-m", "JSONito", "Jot-P", "D2", "TOON", "YAML", "TOML", "JSON-s", "JSON-p"]
+    y-axis "Tokens" 0 --> 16000
+    line "Qwen" [6525, 7748, 7757, 8239, 8292, 8315, 9543, 10180, 11799, 12656]
+    line "Legacy" [6420, 7377, 7794, 7204, 7582, 7079, 7661, 11204, 10966, 11937]
+    line "Claude" [6747, 8132, 8327, 8500, 7928, 8405, 9456, 11485, 12687, 14403]
+```
+<!-- CHART_END -->
+
+### Compact Formats
+
+For machine-to-machine or LLM contexts where readability isn't required.
+
+<!-- COMPACT_START -->
+| Format                                              | Qwen           | Legacy         | Claude         | Bytes          |
+|-----------------------------------------------------|---------------:|---------------:|---------------:|---------------:|
+| **[Jot](jot/)**                                     |   6,525 (-16%) |   6,420 (-13%) |   6,747 (-17%) |  16,621 (-28%) |
+| [JSON](https://www.json.org/) (mini)                |          7,748 |          7,377 |          8,132 |         23,119 |
+| [JSONito](https://github.com/creationix/jsonito)    |    7,757 (+0%) |    7,794 (+6%) |    8,327 (+2%) |  14,059 (-39%) |
+| [D2](https://github.com/creationix/d2)              |    8,292 (+7%) |    7,582 (+3%) |    7,928 (-3%) |  17,328 (-25%) |
+<!-- COMPACT_END -->
+
+### Pretty-Printed Formats
+
+For human-readable output or when LLMs need to read/write structured data.
+
+<!-- PRETTY_START -->
+| Format                                              | Qwen           | Legacy         | Claude         | Bytes          |
+|-----------------------------------------------------|---------------:|---------------:|---------------:|---------------:|
+| **[Jot](jot/) (pretty)**                            |   8,239 (-35%) |   7,204 (-40%) |   8,500 (-41%) |  23,676 (-41%) |
+| [TOON](toon/)                                       |   8,315 (-34%) |   7,079 (-41%) |   8,405 (-42%) |  22,780 (-43%) |
+| [YAML](https://yaml.org/)                           |   9,543 (-25%) |   7,661 (-36%) |   9,456 (-34%) |  26,757 (-33%) |
+| [TOML](https://toml.io/)                            |  10,180 (-20%) |   11,204 (-6%) |  11,485 (-20%) |  28,930 (-27%) |
+| [JSON](json/smart-json.ts) (smart)                  |   11,799 (-7%) |   10,966 (-8%) |  12,687 (-12%) |  32,657 (-18%) |
+| [JSON](https://www.json.org/) (pretty)              |         12,656 |         11,937 |         14,403 |         39,884 |
+<!-- PRETTY_END -->
+
+## Format Descriptions
+
+### [Jot](jot/)
+
+JSON with three optimizations:
+
+1. **Unquoted strings** — omit quotes unless value contains `: ; , { } [ ] "` or parses as number/boolean/null
+2. **Key folding** — `{a:{b:1}}` → `{a.b:1}` for single-key nested objects
+3. **Tables** — `[{a:1},{a:2}]` → `{{:a;1;2}}` for repeating object schemas
+
+```jot
+{config.host:localhost,users:{{:id,name;1,Alice;2,Bob}}}
+```
+
+It also has a pretty-printed variant that adds indentation and newlines for readability.
+
+```jot
+{
+  config.host: localhost,
+  users: {{
+    :id, name;
+      1, Alice;
+      2, Bob
+  }}
+}
+```
+
+### [TOON](toon/)
+
+YAML-like indentation with optional table syntax and count guards.
+
+```toon
+users[2]{id,name}:
+  1,Alice
+  2,Bob
+```
+
+### [JSONito](https://github.com/creationix/jsonito)
+
+Byte-optimized JSON with string deduplication via preamble dictionary.
+
+```jito
+{name'config'version'5~1.0.0enabled'!a~maxRetries6.timeout'eFw.tags'[a~productionapi'v1']}
+```
+
+### [D2](https://github.com/creationix/d2)
+
+Declarative data format using `=` assignment and shell-like quoting.
+
+## Why Not Byte-Optimized Formats?
+
+Formats like JSONito achieve excellent byte compression (-39%) but:
+
+- Token savings are inconsistent (small docs often cost more than JSON)
+- Deduplication preambles add overhead that doesn't scale down
+- LLMs cannot reliably generate formats requiring state tracking
+
+## LLM Encoding Ability
+
+Tested Qwen3-30b's ability to encode JSON → Jot (3 runs per document, 17 docs):
+
+| Document Type                                | Semantic Accuracy |
+|----------------------------------------------|------------------:|
+| Simple configs (small, metrics, package)     |              100% |
+| Key folding test cases                       |              100% |
+| Table-friendly (users-50)                    |              100% |
+| Text-heavy (chat)                            |              100% |
+| Complex/nested (large, firewall, routes)     |                0% |
+| Irregular schemas (medium, hikes, irregular) |                0% |
+| **Overall**                                  |           **47%** |
+
+Small models struggle with Jot's advanced features on complex documents. For LLM-generated output, consider using simpler Jot (unquoted strings only) or providing FORMAT.md as context.
+
+## Tokenizers
+
+- **Qwen**: Qwen3-Coder-30b via LM Studio API
+- **Legacy**: Anthropic legacy tokenizer (`@anthropic-ai/tokenizer`)
+- **Claude**: Claude API token counting endpoint (Sonnet/Opus/Haiku share tokenizer)
+
+## Test Data
+
+18 documents covering diverse structures:
+
+| Document          | Description                      |
+|-------------------|----------------------------------|
+| small             | Config object (6 fields)         |
+| medium            | User list with metadata          |
+| large             | Kubernetes deployment spec       |
+| hikes             | Tabular records (uniform schema) |
+| chat              | LLM conversation (text-heavy)    |
+| metrics           | Time series (numeric-heavy)      |
+| package           | npm manifest (nested deps)       |
+| github-issue      | Mixed nesting with labels        |
+| irregular         | Event log (varying keys)         |
+| users-50          | 50 user records (table-friendly) |
+| logs              | 50 log entries (semi-uniform)    |
+| firewall          | WAF rules (deeply nested)        |
+| products          | E-commerce catalog (variants)    |
+| routes            | API routing config (large tables)|
+| key-folding-*     | Key folding test cases           |
+| json-counts-cache | Cached token counts              |

package/TOKEN_COUNTS.md

@@ -0,0 +1,97 @@
+# Token Counts by Format
+
+## Qwen Tokenizer (LM Studio)
+
+Tokens measured using **Qwen3-Coder-30b** via LM Studio API. This is the primary tokenizer used for testing local model efficiency.
+
+<!-- QWEN_CHART_START -->
+```mermaid
+xychart-beta
+    title "Token Savings vs JSON (negative = better)"
+    x-axis ["Users50", "Hikes", "Medium", "KF-arr", "Firewall", "KF-basic", "Products", "Routes", "Metrics", "Package", "Chat", "KF-mix", "Issue", "Small", "Large", "Irregular", "Logs"]
+    y-axis "% vs JSON" -60 --> 40
+    line "JSON" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    line "Jot" [-48, -29, -27, -26, -19, -19, -19, -15, -14, -12, -12, -12, -10, -8, -8, -7, -3]
+    line "YAML" [25, 18, 27, 19, 24, 10, 26, 16, 20, 7, 8, 14, 11, 17, 23, 28, 18]
+    line "TOON" [-40, -23, -14, 9, 30, 9, 10, 8, -6, 7, -11, 13, 2, 4, 18, 29, 18]
+```
+<!-- QWEN_CHART_END -->
+
+### Per-File Breakdown (Qwen)
+
+<!-- QWEN_TABLE_START -->
+| Format | Chat | Firewall | Issue | Hikes | Irregular | KF-basic | KF-mix | KF-arr | Large | Logs | Medium | Metrics | Package | Products | Routes | Small | Users50 | Total |
+|--------|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|
+| Jot | [67](jot/chat.jot) | [669](jot/firewall.jot) | [79](jot/github-issue.jot) | [112](jot/hikes.jot) | [63](jot/irregular.jot) | [47](jot/key-folding-basic.jot) | [69](jot/key-folding-mixed.jot) | [43](jot/key-folding-with-array.jot) | [246](jot/large.jot) | [2044](jot/logs.jot) | [71](jot/medium.jot) | [101](jot/metrics.jot) | [85](jot/package.jot) | [702](jot/products.jot) | [1246](jot/routes.jot) | [44](jot/small.jot) | [663](jot/users-50.jot) | 6,351 |
+| JSONito | [86](jsonito/chat.jito) | [919](jsonito/firewall.jito) | [89](jsonito/github-issue.jito) | [158](jsonito/hikes.jito) | [59](jsonito/irregular.jito) | [62](jsonito/key-folding-basic.jito) | [82](jsonito/key-folding-mixed.jito) | [63](jsonito/key-folding-with-array.jito) | [312](jsonito/large.jito) | [1941](jsonito/logs.jito) | [103](jsonito/medium.jito) | [89](jsonito/metrics.jito) | [101](jsonito/package.jito) | [846](jsonito/products.jito) | [1426](jsonito/routes.jito) | [45](jsonito/small.jito) | [1234](jsonito/users-50.jito) | 7,615 |
+| JSON-m | [76](json/chat.json) | [827](json/firewall.json) | [88](json/github-issue.json) | [158](json/hikes.json) | [68](json/irregular.json) | [58](json/key-folding-basic.json) | [78](json/key-folding-mixed.json) | [58](json/key-folding-with-array.json) | [266](json/large.json) | [2108](json/logs.json) | [97](json/medium.json) | [117](json/metrics.json) | [97](json/package.json) | [866](json/products.json) | [1459](json/routes.json) | [48](json/small.json) | [1279](json/users-50.json) | 7,748 |
+| Jot-P | [73](jot/chat.pretty.jot) | [1119](jot/firewall.pretty.jot) | [105](jot/github-issue.pretty.jot) | [142](jot/hikes.pretty.jot) | [91](jot/irregular.pretty.jot) | [54](jot/key-folding-basic.pretty.jot) | [96](jot/key-folding-mixed.pretty.jot) | [52](jot/key-folding-with-array.pretty.jot) | [365](jot/large.pretty.jot) | [2304](jot/logs.pretty.jot) | [100](jot/medium.pretty.jot) | [123](jot/metrics.pretty.jot) | [120](jot/package.pretty.jot) | [828](jot/products.pretty.jot) | [1521](jot/routes.pretty.jot) | [59](jot/small.pretty.jot) | [863](jot/users-50.pretty.jot) | 8,015 |
+| D2 | [80](d2/chat.d2) | [894](d2/firewall.d2) | [97](d2/github-issue.d2) | [173](d2/hikes.d2) | [81](d2/irregular.d2) | [60](d2/key-folding-basic.d2) | [80](d2/key-folding-mixed.d2) | [67](d2/key-folding-with-array.d2) | [316](d2/large.d2) | [2092](d2/logs.d2) | [104](d2/medium.d2) | [138](d2/metrics.d2) | [90](d2/package.d2) | [994](d2/products.d2) | [1536](d2/routes.d2) | [55](d2/small.d2) | [1202](d2/users-50.d2) | 8,059 |
+| TOON | [68](toon/chat.toon) | [1073](toon/firewall.toon) | [90](toon/github-issue.toon) | [122](toon/hikes.toon) | [88](toon/irregular.toon) | [63](toon/key-folding-basic.toon) | [88](toon/key-folding-mixed.toon) | [63](toon/key-folding-with-array.toon) | [313](toon/large.toon) | [2492](toon/logs.toon) | [83](toon/medium.toon) | [110](toon/metrics.toon) | [104](toon/package.toon) | [954](toon/products.toon) | [1574](toon/routes.toon) | [50](toon/small.toon) | [763](toon/users-50.toon) | 8,098 |
+| YAML | [82](yaml/chat.yaml) | [1029](yaml/firewall.yaml) | [98](yaml/github-issue.yaml) | [187](yaml/hikes.yaml) | [87](yaml/irregular.yaml) | [64](yaml/key-folding-basic.yaml) | [89](yaml/key-folding-mixed.yaml) | [69](yaml/key-folding-with-array.yaml) | [327](yaml/large.yaml) | [2487](yaml/logs.yaml) | [123](yaml/medium.yaml) | [140](yaml/metrics.yaml) | [104](yaml/package.yaml) | [1095](yaml/products.yaml) | [1696](yaml/routes.yaml) | [56](yaml/small.yaml) | [1597](yaml/users-50.yaml) | 9,330 |
+| TOML | [84](toml/chat.toml) | [1495](toml/firewall.toml) | [99](toml/github-issue.toml) | [189](toml/hikes.toml) | [86](toml/irregular.toml) | [60](toml/key-folding-basic.toml) | [85](toml/key-folding-mixed.toml) | [61](toml/key-folding-with-array.toml) | [377](toml/large.toml) | [2498](toml/logs.toml) | [118](toml/medium.toml) | [139](toml/metrics.toml) | [104](toml/package.toml) | [1114](toml/products.toml) | [1790](toml/routes.toml) | [56](toml/small.toml) | [1625](toml/users-50.toml) | 9,980 |
+<!-- QWEN_TABLE_END -->
+
+---
+
+## Legacy Claude Tokenizer
+
+Tokens measured using **@anthropic-ai/tokenizer** (Claude's legacy tokenizer). This is the older tokenizer used by earlier Claude models.
+
+<!-- LEGACY_CHART_START -->
+```mermaid
+xychart-beta
+    title "Token Savings vs JSON (negative = better)"
+    x-axis ["Users50", "Hikes", "Products", "Metrics", "Medium", "Routes", "Chat", "KF-arr", "KF-basic", "Large", "KF-mix", "Issue", "Firewall", "Package", "Logs", "Small", "Irregular"]
+    y-axis "% vs JSON" -50 --> 30
+    line "JSON" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    line "Jot" [-37, -26, -20, -20, -20, -11, -10, -10, -8, -8, -6, -6, -2, -2, -1, 0, 2]
+    line "YAML" [3, 3, 4, 5, 4, 4, 4, 15, 8, 9, 4, 7, 8, 0, 2, 11, 8]
+    line "TOON" [-37, -24, -7, -21, -20, 3, -11, 12, 6, 4, 6, 0, 13, 0, 8, 3, 10]
+```
+<!-- LEGACY_CHART_END -->
+
+### Per-File Breakdown (Legacy)
+
+<!-- LEGACY_TABLE_START -->
+| Format | Chat | Metrics | Large | KF-mix | Logs | Firewall | Small | Issue | Users50 | Medium | Hikes | Package | KF-basic | Irregular | KF-arr | Products | Routes | Total |
+|--------|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|
+| Jot | [63](jot/chat.jot) | [60](jot/metrics.jot) | [221](jot/large.jot) | [65](jot/key-folding-mixed.jot) | [1737](jot/logs.jot) | [825](jot/firewall.jot) | [36](jot/small.jot) | [68](jot/github-issue.jot) | [837](jot/users-50.jot) | [76](jot/medium.jot) | [103](jot/hikes.jot) | [91](jot/package.jot) | [45](jot/key-folding-basic.jot) | [49](jot/irregular.jot) | [47](jot/key-folding-with-array.jot) | [613](jot/products.jot) | [1352](jot/routes.jot) | 6,288 |
+| JSON-m | [70](json/chat.json) | [75](json/metrics.json) | [239](json/large.json) | [69](json/key-folding-mixed.json) | [1750](json/logs.json) | [845](json/firewall.json) | [36](json/small.json) | [72](json/github-issue.json) | [1326](json/users-50.json) | [95](json/medium.json) | [139](json/hikes.json) | [93](json/package.json) | [49](json/key-folding-basic.json) | [48](json/irregular.json) | [52](json/key-folding-with-array.json) | [771](json/products.json) | [1516](json/routes.json) | 7,245 |
+| YAML | [73](yaml/chat.yaml) | [79](yaml/metrics.yaml) | [260](yaml/large.yaml) | [72](yaml/key-folding-mixed.yaml) | [1780](yaml/logs.yaml) | [911](yaml/firewall.yaml) | [40](yaml/small.yaml) | [77](yaml/github-issue.yaml) | [1365](yaml/users-50.yaml) | [99](yaml/medium.yaml) | [143](yaml/hikes.yaml) | [93](yaml/package.yaml) | [53](yaml/key-folding-basic.yaml) | [52](yaml/irregular.yaml) | [60](yaml/key-folding-with-array.yaml) | [799](yaml/products.yaml) | [1572](yaml/routes.yaml) | 7,528 |
+| TOON | [62](toon/chat.toon) | [59](toon/metrics.toon) | [249](toon/large.toon) | [73](toon/key-folding-mixed.toon) | [1885](toon/logs.toon) | [955](toon/firewall.toon) | [37](toon/small.toon) | [72](toon/github-issue.toon) | [836](toon/users-50.toon) | [76](toon/medium.toon) | [105](toon/hikes.toon) | [93](toon/package.toon) | [52](toon/key-folding-basic.toon) | [53](toon/irregular.toon) | [58](toon/key-folding-with-array.toon) | [720](toon/products.toon) | [1557](toon/routes.toon) | 6,942 |
+<!-- LEGACY_TABLE_END -->
+
+Note: The legacy tokenizer may produce different results than modern Claude models, but is useful for comparison and runs locally without API calls.
+
+---
+
+## Modern Claude Tokenizer
+
+Tokens measured using **Claude API** token counting endpoint (claude-sonnet-4). This represents the actual token usage for modern Claude models. Token counting is free via the API.
+
+<!-- CLAUDE_CHART_START -->
+```mermaid
+xychart-beta
+    title "Token Savings vs JSON (negative = better)"
+    x-axis ["Users50", "Hikes", "Products", "KF-basic", "KF-arr", "Metrics", "Medium", "Large", "Firewall", "KF-mix", "Routes", "Chat", "Issue", "Irregular", "Package", "Small", "Logs"]
+    y-axis "% vs JSON" -50 --> 40
+    line "JSON" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    line "Jot" [-39, -29, -24, -24, -24, -23, -22, -16, -15, -15, -14, -12, -9, -6, -6, -4, -2]
+    line "YAML" [22, 19, 27, 1, 14, 19, 19, 16, 13, 6, 9, 6, 8, 21, 4, 13, 18]
+    line "TOON" [-36, -25, 8, 1, 6, -18, -14, 12, 17, 8, 7, -10, 1, 24, 5, 2, 21]
+```
+<!-- CLAUDE_CHART_END -->
+
+### Per-File Breakdown (Claude)
+
+<!-- CLAUDE_TABLE_START -->
+| Format | Chat | Metrics | Large | KF-mix | Logs | Firewall | Small | Issue | Users50 | Medium | Hikes | Package | KF-basic | Irregular | KF-arr | Products | Routes | Total |
+|--------|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|------:|
+| Jot | [71](jot/chat.jot) | [68](jot/metrics.jot) | [241](jot/large.jot) | [76](jot/key-folding-mixed.jot) | [1762](jot/logs.jot) | [867](jot/firewall.jot) | [44](jot/small.jot) | [78](jot/github-issue.jot) | [830](jot/users-50.jot) | [83](jot/medium.jot) | [115](jot/hikes.jot) | [99](jot/package.jot) | [53](jot/key-folding-basic.jot) | [59](jot/irregular.jot) | [53](jot/key-folding-with-array.jot) | [634](jot/products.jot) | [1472](jot/routes.jot) | 6,605 |
+| JSON-m | [81](json/chat.json) | [88](json/metrics.json) | [286](json/large.json) | [89](json/key-folding-mixed.json) | [1798](json/logs.json) | [1022](json/firewall.json) | [46](json/small.json) | [86](json/github-issue.json) | [1369](json/users-50.json) | [107](json/medium.json) | [163](json/hikes.json) | [105](json/package.json) | [70](json/key-folding-basic.json) | [63](json/irregular.json) | [70](json/key-folding-with-array.json) | [839](json/products.json) | [1707](json/routes.json) | 7,989 |
+| YAML | [86](yaml/chat.yaml) | [105](yaml/metrics.yaml) | [333](yaml/large.yaml) | [94](yaml/key-folding-mixed.yaml) | [2118](yaml/logs.yaml) | [1152](yaml/firewall.yaml) | [52](yaml/small.yaml) | [93](yaml/github-issue.yaml) | [1666](yaml/users-50.yaml) | [127](yaml/medium.yaml) | [194](yaml/hikes.yaml) | [109](yaml/package.yaml) | [71](yaml/key-folding-basic.yaml) | [76](yaml/irregular.yaml) | [80](yaml/key-folding-with-array.yaml) | [1063](yaml/products.yaml) | [1857](yaml/routes.yaml) | 9,276 |
+| TOON | [73](toon/chat.toon) | [72](toon/metrics.toon) | [319](toon/large.toon) | [96](toon/key-folding-mixed.toon) | [2175](toon/logs.toon) | [1195](toon/firewall.toon) | [47](toon/small.toon) | [87](toon/github-issue.toon) | [879](toon/users-50.toon) | [92](toon/medium.toon) | [123](toon/hikes.toon) | [110](toon/package.toon) | [71](toon/key-folding-basic.toon) | [78](toon/irregular.toon) | [74](toon/key-folding-with-array.toon) | [907](toon/products.toon) | [1822](toon/routes.toon) | 8,220 |
+<!-- CLAUDE_TABLE_END -->
+
+Note: Run `ANTHROPIC_API_KEY=... bun scripts/count-claude-tokens.ts` to regenerate Claude counts.

package/bun.lock

@@ -0,0 +1,19 @@
+{
+  "lockfileVersion": 1,
+  "workspaces": {
+    "": {
+      "devDependencies": {
+        "@types/bun": "^1.3.5",
+      },
+    },
+  },
+  "packages": {
+    "@types/bun": ["@types/bun@1.3.5", "", { "dependencies": { "bun-types": "1.3.5" } }, "sha512-RnygCqNrd3srIPEWBd5LFeUYG7plCoH2Yw9WaZGyNmdTEei+gWaHqydbaIRkIkcbXwhBT94q78QljxN0Sk838w=="],
+
+    "@types/node": ["@types/node@25.0.6", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-NNu0sjyNxpoiW3YuVFfNz7mxSQ+S4X2G28uqg2s+CzoqoQjLPsWSbsFFyztIAqt2vb8kfEAsJNepMGPTxFDx3Q=="],
+
+    "bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="],
+
+    "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
+  }
+}

package/jot.test.ts

@@ -0,0 +1,133 @@
+import { describe, test, expect } from "bun:test"
+import { stringify, parse } from "./jot"
+
+describe("stringify", () => {
+  test("null", () => expect(stringify(null)).toBe("null"))
+  test("true", () => expect(stringify(true)).toBe("true"))
+  test("false", () => expect(stringify(false)).toBe("false"))
+  test("number", () => expect(stringify(42)).toBe("42"))
+  test("float", () => expect(stringify(3.14)).toBe("3.14"))
+
+  describe("strings", () => {
+    test("simple string", () => expect(stringify("hello")).toBe("hello"))
+    test("string with space", () => expect(stringify("hello world")).toBe("hello world"))
+    test("empty string", () => expect(stringify("")).toBe('""'))
+    test("numeric string", () => expect(stringify("123")).toBe('"123"'))
+    test("reserved word", () => expect(stringify("true")).toBe('"true"'))
+    test("contains colon", () => expect(stringify("a:b")).toBe('"a:b"'))
+    test("contains semicolon", () => expect(stringify("a;b")).toBe('"a;b"'))
+  })
+
+  describe("arrays", () => {
+    test("empty array", () => expect(stringify([])).toBe("[]"))
+    test("simple array", () => expect(stringify([1, 2, 3])).toBe("[1,2,3]"))
+    test("string array", () => expect(stringify(["a", "b"])).toBe("[a,b]"))
+  })
+
+  describe("objects", () => {
+    test("empty object", () => expect(stringify({})).toBe("{}"))
+    test("simple object", () => expect(stringify({ name: "Alice", age: 30 })).toBe("{name:Alice,age:30}"))
+  })
+
+  describe("key folding", () => {
+    test("fold 1 level", () => expect(stringify({ a: { b: 1 } })).toBe("{a.b:1}"))
+    test("fold 2 levels", () => expect(stringify({ a: { b: { c: 1 } } })).toBe("{a.b.c:1}"))
+    test("no fold multi-key", () => expect(stringify({ a: { b: 1, c: 2 } })).toBe("{a:{b:1,c:2}}"))
+    test("key with dot", () => expect(stringify({ "a.b": 1 })).toBe('{"a.b":1}'))
+    test("key with dot nested", () => expect(stringify({ "a.b": { c: 1 } })).toBe('{"a.b":{c:1}}'))
+  })
+
+  describe("tables", () => {
+    test("uniform table", () => expect(stringify([{ a: 1, b: 2 }, { a: 3, b: 4 }])).toBe("{{:a,b;1,2;3,4}}"))
+    test("3-row table", () => expect(stringify([{ x: 1 }, { x: 2 }, { x: 3 }])).toBe("{{:x;1;2;3}}"))
+    test("mixed schema with reuse", () => expect(stringify([{ a: 1 }, { a: 2 }, { b: 3 }])).toBe("{{:a;1;2;:b;3}}"))
+    test("no reuse", () => expect(stringify([{ a: 1 }, { b: 2 }])).toBe("[{a:1},{b:2}]"))
+    test("single obj", () => expect(stringify([{ a: 1 }])).toBe("[{a:1}]"))
+  })
+})
+
+describe("parse", () => {
+  test("null", () => expect(parse("null")).toBe(null))
+  test("true", () => expect(parse("true")).toBe(true))
+  test("number", () => expect(parse("42")).toBe(42))
+  test("string", () => expect(parse("hello")).toBe("hello"))
+  test("quoted", () => expect(parse('"hello world"')).toBe("hello world"))
+
+  test("array", () => expect(parse("[1,2,3]")).toEqual([1, 2, 3]))
+  test("object", () => expect(parse("{name:Alice,age:30}")).toEqual({ name: "Alice", age: 30 }))
+
+  describe("key unfolding", () => {
+    test("fold", () => expect(parse("{a.b:1}")).toEqual({ a: { b: 1 } }))
+    test("fold deep", () => expect(parse("{a.b.c:1}")).toEqual({ a: { b: { c: 1 } } }))
+    test("quoted dot key", () => expect(parse('{"a.b":1}')).toEqual({ "a.b": 1 }))
+  })
+
+  describe("tables", () => {
+    test("table", () => expect(parse("{{:a,b;1,2;3,4}}")).toEqual([{ a: 1, b: 2 }, { a: 3, b: 4 }]))
+    test("schema change", () => expect(parse("{{:a;1;:b;2}}")).toEqual([{ a: 1 }, { b: 2 }]))
+  })
+})
+
+describe("round-trip", () => {
+  const testCases: [string, unknown][] = [
+    ["null", null],
+    ["true", true],
+    ["false", false],
+    ["number", 42],
+    ["string", "hello"],
+    ["empty array", []],
+    ["number array", [1, 2, 3]],
+    ["empty object", {}],
+    ["simple object", { a: 1, b: 2 }],
+    ["nested 1 level", { a: { b: 1 } }],
+    ["nested 2 levels", { a: { b: { c: 1 } } }],
+    ["uniform table", [{ a: 1, b: 2 }, { a: 3, b: 4 }]],
+    ["mixed schema", [{ a: 1 }, { b: 2 }]],
+    ["nested table", { users: [{ id: 1, name: "Alice" }, { id: 2, name: "Bob" }] }],
+    ["key with dot", { "a.b": 1 }],
+    ["key with semicolon", { "x;y": "test" }],
+    ["string with semicolon", "a;b"],
+  ]
+
+  for (const [name, original] of testCases) {
+    test(name, () => {
+      const encoded = stringify(original)
+      const decoded = parse(encoded)
+      expect(decoded).toEqual(original)
+    })
+  }
+})
+
+describe("samples", () => {
+  const { readdirSync, readFileSync } = require("fs")
+  const { join } = require("path")
+
+  const samplesDir = join(__dirname, "samples")
+  const jsonFiles = readdirSync(samplesDir)
+    .filter((f: string) => f.endsWith(".json") && !f.includes(".pretty."))
+
+  for (const jsonFile of jsonFiles) {
+    const baseName = jsonFile.replace(".json", "")
+    const jotFile = `${baseName}.jot`
+
+    describe(baseName, () => {
+      const jsonPath = join(samplesDir, jsonFile)
+      const jotPath = join(samplesDir, jotFile)
+
+      const jsonContent = readFileSync(jsonPath, "utf-8").trim()
+      const expectedJot = readFileSync(jotPath, "utf-8").trim()
+      const originalData = JSON.parse(jsonContent)
+
+      test("encoding matches jot", () => {
+        const encoded = stringify(originalData)
+        expect(encoded).toBe(expectedJot)
+      })
+
+      test("round-trip matches json semantically", () => {
+        const encoded = stringify(originalData)
+        const decoded = parse(encoded)
+        expect(decoded).toEqual(originalData)
+      })
+    })
+  }
+})