@jarenjs/josl 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/FORMAT.md +235 -0
- package/README.md +494 -0
- package/dist/types/cst.d.ts +78 -0
- package/dist/types/csv-machine.d.ts +104 -0
- package/dist/types/csv-stream.d.ts +102 -0
- package/dist/types/csv.d.ts +141 -0
- package/dist/types/errors.d.ts +79 -0
- package/dist/types/gbnf.d.ts +20 -0
- package/dist/types/index.d.ts +12 -0
- package/dist/types/jsonx-scalar.d.ts +92 -0
- package/dist/types/jsonx-stream.d.ts +163 -0
- package/dist/types/jsonx.d.ts +31 -0
- package/dist/types/machine.d.ts +97 -0
- package/dist/types/parse.d.ts +24 -0
- package/dist/types/stream.d.ts +32 -0
- package/dist/types/stringify.d.ts +63 -0
- package/dist/types/util.d.ts +56 -0
- package/dist/types/values.d.ts +60 -0
- package/dist/types/write.d.ts +92 -0
- package/package.json +104 -0
- package/schemas/jaren-josl-data.schema.json +21 -0
- package/src/cst.js +256 -0
- package/src/csv-machine.js +908 -0
- package/src/csv-stream.js +196 -0
- package/src/csv.js +363 -0
- package/src/errors.js +103 -0
- package/src/gbnf.js +179 -0
- package/src/index.js +50 -0
- package/src/jsonx-scalar.js +326 -0
- package/src/jsonx-stream.js +806 -0
- package/src/jsonx.js +342 -0
- package/src/machine.js +1252 -0
- package/src/parse.js +37 -0
- package/src/stream.js +57 -0
- package/src/stringify.js +341 -0
- package/src/util.js +96 -0
- package/src/values.js +104 -0
- package/src/write.js +226 -0
package/README.md
ADDED
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
# @jarenjs/josl
|
|
2
|
+
|
|
3
|
+
**JOSL — JavaScript Obvious Streaming Language.**
|
|
4
|
+
TOML 1.0, backward compatible, extended with JavaScript's obvious value
|
|
5
|
+
types (`null`, bigint, regexp, datetimes) and a streamable `[[]]` root
|
|
6
|
+
array — plus **JSONX**, the same extensions over JSON, and a **CSV**
|
|
7
|
+
reader/writer that heals damaged input instead of guessing at it. Built for
|
|
8
|
+
LLM-to-LLM pipelines: chunk-feedable parsing, document-order events,
|
|
9
|
+
machine-repairable errors with line/column and a `hint`.
|
|
10
|
+
|
|
11
|
+
Zero dependencies, vanilla JS, CSP-safe, tree-shakeable subpath exports.
|
|
12
|
+
See [FORMAT.md](./FORMAT.md) for the language definition and rationale.
|
|
13
|
+
|
|
14
|
+
## Quick start
|
|
15
|
+
|
|
16
|
+
```js
|
|
17
|
+
import { parseJosl, stringifyJosl } from '@jarenjs/josl';
|
|
18
|
+
|
|
19
|
+
const doc = parseJosl(`
|
|
20
|
+
title = "example"
|
|
21
|
+
big = 9007199254740993n # bigint literal
|
|
22
|
+
match = /^ok$/i # regexp literal
|
|
23
|
+
middle-name = null # TOML has no null; JOSL does
|
|
24
|
+
when = 2026-07-18T12:00:00Z # a real Date
|
|
25
|
+
|
|
26
|
+
[server]
|
|
27
|
+
host = "localhost"
|
|
28
|
+
`);
|
|
29
|
+
|
|
30
|
+
stringifyJosl(doc); // round-trips
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Strict TOML in and out of the same engine:
|
|
34
|
+
|
|
35
|
+
```js
|
|
36
|
+
import { parseToml, stringifyToml } from '@jarenjs/josl';
|
|
37
|
+
|
|
38
|
+
parseToml(tomlText); // rejects JOSL extensions
|
|
39
|
+
stringifyToml(doc, { onNull: 'omit' }); // downlevels a JOSL value
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Streaming (the point)
|
|
43
|
+
|
|
44
|
+
Two incremental readers share one API shape (`feed`/`end`/`root`) and
|
|
45
|
+
one `pair` event, so a consumer keyed on paths never branches on
|
|
46
|
+
syntax. Chunks may split ANY token — escapes mid-`\uXXXX`, numbers,
|
|
47
|
+
`tru` + `e` — which is what makes token-by-token LLM output feedable.
|
|
48
|
+
|
|
49
|
+
**JOSL / TOML** (line-oriented):
|
|
50
|
+
|
|
51
|
+
```js
|
|
52
|
+
import { createStreamReader } from '@jarenjs/josl/stream';
|
|
53
|
+
|
|
54
|
+
const reader = createStreamReader({
|
|
55
|
+
onEvent(e) {
|
|
56
|
+
// document order, absolute paths, fired the moment a line completes
|
|
57
|
+
if (e.type === 'root-item') console.log('record', e.index, 'started');
|
|
58
|
+
if (e.type === 'pair') console.log(e.path.join('/'), '=', e.value);
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
for await (const chunk of llmTokenStream) // chunks may split ANY token
|
|
63
|
+
reader.feed(chunk);
|
|
64
|
+
const records = reader.end();
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
**JSONX / strict JSON** (nested):
|
|
68
|
+
|
|
69
|
+
```js
|
|
70
|
+
import { createJsonxStreamReader } from '@jarenjs/josl/jsonx-stream';
|
|
71
|
+
|
|
72
|
+
const reader = createJsonxStreamReader({
|
|
73
|
+
mode: 'json', // strict: rejects every JSONX extension, matches JSON.parse
|
|
74
|
+
onEvent(e) {
|
|
75
|
+
if (e.type === 'pair') console.log(e.path.join('/'), '=', e.value);
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
reader.feed('{"run": [{"i": 1, "ops": 6'); // any split point works
|
|
79
|
+
reader.feed('1200}]}');
|
|
80
|
+
const doc = reader.end(); // { run: [{ i: 1, ops: 61200 }] }
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The unified event vocabulary:
|
|
84
|
+
|
|
85
|
+
| event | reader | fields | when |
|
|
86
|
+
| --- | --- | --- | --- |
|
|
87
|
+
| `pair` | both | `path`, `key`, `value`, `line` | a scalar member completes |
|
|
88
|
+
| `item` | jsonx | `path`, `index`, `value`, `line` | a scalar array element completes |
|
|
89
|
+
| `object-start` / `array-start` | jsonx | `path`, `line` | `{` / `[` opened |
|
|
90
|
+
| `object-end` / `array-end` | jsonx | `path`, `value`, `line` | `}` / `]` closed |
|
|
91
|
+
| `table` / `table-array` / `root-item` | josl | `path`, `line` (+`index`) | a header line completes |
|
|
92
|
+
| `text-partial` | jsonx | `path`, `text`, `line` | more of a string arrived (opt-in) |
|
|
93
|
+
|
|
94
|
+
Paths are absolute (strings for keys, numbers for indices), so events
|
|
95
|
+
are directly JSON-Pointer-able. In the JSONX reader a container value
|
|
96
|
+
does NOT additionally fire `pair`/`item` — its start/end events carry
|
|
97
|
+
that; in the line-oriented JOSL reader inline tables arrive as
|
|
98
|
+
completed `pair` values and containers have no end events.
|
|
99
|
+
|
|
100
|
+
### Documents larger than memory
|
|
101
|
+
|
|
102
|
+
Feeding a document in chunks bounds the *parse*, not the *result*: the
|
|
103
|
+
reader still ends up holding everything it has read. For a continent-
|
|
104
|
+
sized `FeatureCollection` or a million-line log that is the wrong
|
|
105
|
+
answer, and it is the reason `detach` exists.
|
|
106
|
+
|
|
107
|
+
A value whose absolute path matches the pattern is **never linked into
|
|
108
|
+
the tree**. Its completion event still carries the whole thing, so the
|
|
109
|
+
consumer sees every record exactly once — but letting go of the event
|
|
110
|
+
lets go of the record.
|
|
111
|
+
|
|
112
|
+
```js
|
|
113
|
+
const reader = createJsonxStreamReader({
|
|
114
|
+
mode: 'json',
|
|
115
|
+
detach: ['features', '*'], // '*' matches any one segment
|
|
116
|
+
onEvent(e) {
|
|
117
|
+
if (e.type === 'object-end' && e.path.length === 2)
|
|
118
|
+
consume(e.value); // a whole GeoJSON Feature
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
for await (const chunk of fileChunks) reader.feed(chunk);
|
|
123
|
+
reader.end(); // { type: 'FeatureCollection', name: '…', features: [] }
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
`root()` comes back holding the document's *frame* — its header members,
|
|
127
|
+
and an empty array where the records would have been — however many
|
|
128
|
+
records went past. The pattern matches an exact path, not a prefix, so a
|
|
129
|
+
feature's own rings are not separately detached: they belong to their
|
|
130
|
+
feature and are freed with it.
|
|
131
|
+
|
|
132
|
+
Measured on a synthetic OpenStreetMap-shaped extract (`node --expose-gc
|
|
133
|
+
benchmark/jsonx-stream.js`), reading 20 000 features of 40 vertices each:
|
|
134
|
+
|
|
135
|
+
| | peak live set | root holds |
|
|
136
|
+
| --- | --- | --- |
|
|
137
|
+
| default | 185.4 MB | 20 000 features |
|
|
138
|
+
| `detach: ['features','*']` | < 0.1 MB | 0 |
|
|
139
|
+
|
|
140
|
+
and the detached figure does not move at 80 000 features — the peak is
|
|
141
|
+
the 64 kB feed buffer plus one feature at a time, not the document.
|
|
142
|
+
"Peak live set" means the heap after a forced collection: sampling
|
|
143
|
+
`heapUsed` without collecting first measures how lazily V8 sweeps, which
|
|
144
|
+
grows with the heap and would make even a detached read look like it
|
|
145
|
+
accumulates.
|
|
146
|
+
|
|
147
|
+
### Progressive text
|
|
148
|
+
|
|
149
|
+
Scalars normally fire once, on completion — but a long string in an LLM
|
|
150
|
+
response is worth showing as it arrives. `partialText: true` adds
|
|
151
|
+
`text-partial` events carrying the *delta* since the previous one,
|
|
152
|
+
already unescaped:
|
|
153
|
+
|
|
154
|
+
```js
|
|
155
|
+
const reader = createJsonxStreamReader({
|
|
156
|
+
partialText: true,
|
|
157
|
+
onEvent(e) {
|
|
158
|
+
if (e.type === 'text-partial') process.stdout.write(e.text); // append
|
|
159
|
+
if (e.type === 'pair') console.log('\ncomplete:', e.path.join('/'));
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
A value's deltas add up to exactly its string — nothing is left over in
|
|
165
|
+
the closing `pair`/`item` event, which still carries the whole value as
|
|
166
|
+
the completion signal. Deltas never split an escape or a surrogate pair,
|
|
167
|
+
so appending them to a display is always safe. Object keys emit none: a
|
|
168
|
+
key has no path until it is complete.
|
|
169
|
+
|
|
170
|
+
### Streaming charts with @jarenjs/charts
|
|
171
|
+
|
|
172
|
+
`@jarenjs/charts`' stream adapter consumes these events directly (it
|
|
173
|
+
never imports a parser — pair them at the call site):
|
|
174
|
+
|
|
175
|
+
```js
|
|
176
|
+
import { createJsonxStreamReader } from '@jarenjs/josl/jsonx-stream';
|
|
177
|
+
import { createStreamAdapter } from '@jarenjs/charts/stream-adapter';
|
|
178
|
+
import { compileChart } from '@jarenjs/charts';
|
|
179
|
+
|
|
180
|
+
const adapter = createStreamAdapter('line', {
|
|
181
|
+
recordPath: ['run'], xField: 'i', yField: 'ops', maxPoints: 200,
|
|
182
|
+
});
|
|
183
|
+
const reader = createJsonxStreamReader({ mode: 'json', onEvent: adapter.onEvent });
|
|
184
|
+
|
|
185
|
+
for await (const chunk of feed) {
|
|
186
|
+
reader.feed(chunk);
|
|
187
|
+
render(compileChart({ type: 'line', title: 'live' }, adapter.getData()).toVnode());
|
|
188
|
+
}
|
|
189
|
+
reader.end();
|
|
190
|
+
adapter.endDocument();
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
An LLM emitting a list of records streams naturally:
|
|
194
|
+
|
|
195
|
+
```toml
|
|
196
|
+
[[]]
|
|
197
|
+
name = "first"
|
|
198
|
+
score = 0.92
|
|
199
|
+
|
|
200
|
+
[[]]
|
|
201
|
+
name = "second"
|
|
202
|
+
score = 0.87
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Each `[[]]` completes the previous record — `reader.root()` exposes the
|
|
206
|
+
partial array at any time, so record *N* can be processed while the
|
|
207
|
+
model is still emitting record *N+1*.
|
|
208
|
+
|
|
209
|
+
## Streaming writer
|
|
210
|
+
|
|
211
|
+
The write-side mirror of the reader — build a document event by event and
|
|
212
|
+
ship each chunk as it is produced:
|
|
213
|
+
|
|
214
|
+
```js
|
|
215
|
+
import { createStreamWriter, stringifyJoslChunks } from '@jarenjs/josl/write';
|
|
216
|
+
|
|
217
|
+
const w = createStreamWriter({ onChunk: (c) => response.write(c) });
|
|
218
|
+
w.pair('title', 'run 42')
|
|
219
|
+
.table('server')
|
|
220
|
+
.pair('host', 'localhost');
|
|
221
|
+
// root-array documents: w.rootItem(record) per completed record
|
|
222
|
+
const text = w.end();
|
|
223
|
+
|
|
224
|
+
// or stream an existing value, one chunk per [[]] record:
|
|
225
|
+
for (const chunk of stringifyJoslChunks(records))
|
|
226
|
+
response.write(chunk);
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
The writer validates what the reader would reject (duplicate keys and
|
|
230
|
+
headers, root table/array mixing, TOML downleveling) and shares its
|
|
231
|
+
serialization with `stringifyJosl`, so both produce identical text.
|
|
232
|
+
|
|
233
|
+
## Editing a document (CST)
|
|
234
|
+
|
|
235
|
+
`parseJosl` + `stringifyJosl` round-trips *data*. When the document
|
|
236
|
+
itself matters — a config file a human wrote, with comments and
|
|
237
|
+
alignment — parse it as a CST instead. Reprinting an untouched document
|
|
238
|
+
returns the original bytes; an edit changes only the value's own bytes:
|
|
239
|
+
|
|
240
|
+
```js
|
|
241
|
+
import { parseTomlCst } from '@jarenjs/josl/cst';
|
|
242
|
+
|
|
243
|
+
const doc = parseTomlCst(readFileSync('config.toml', 'utf8'));
|
|
244
|
+
doc.set(['server', 'port'], 9090);
|
|
245
|
+
writeFileSync('config.toml', doc.toString());
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
```diff
|
|
249
|
+
[server]
|
|
250
|
+
host = "localhost" # keep me
|
|
251
|
+
- port = 8080
|
|
252
|
+
+ port = 9090
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
`get`/`set`/`delete` take absolute paths (numbers index an array of
|
|
256
|
+
tables); `set` on a key that does not exist appends it to the section
|
|
257
|
+
its path names, not to the end of the file. `toJSON()` gives the value
|
|
258
|
+
model, recomputed from the text after an edit so it can never drift
|
|
259
|
+
from the bytes. Byte-identical reprinting is verified against every
|
|
260
|
+
document in the official toml-test valid corpus.
|
|
261
|
+
|
|
262
|
+
## JSONX
|
|
263
|
+
|
|
264
|
+
```js
|
|
265
|
+
import { parseJsonx, stringifyJsonx } from '@jarenjs/josl/jsonx';
|
|
266
|
+
|
|
267
|
+
parseJsonx('{"big": 123n, "re": /a+/g, "when": 2026-07-18}');
|
|
268
|
+
|
|
269
|
+
// document-order events instead of JSON.parse's bottom-up reviver:
|
|
270
|
+
parseJsonx(text, { onEvent: (e) => console.log(e.type, e.path, e.value) });
|
|
271
|
+
|
|
272
|
+
parseJsonx(text, { mode: 'json' }); // bit-compatible JSON.parse
|
|
273
|
+
stringifyJsonx(v, { mode: 'json' }); // delegates to JSON.stringify
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
## Exports
|
|
277
|
+
|
|
278
|
+
| Subpath | What |
|
|
279
|
+
| --- | --- |
|
|
280
|
+
| `@jarenjs/josl/parse` | `parseJosl`, `parseToml` |
|
|
281
|
+
| `@jarenjs/josl/cst` | `parseJoslCst`, `parseTomlCst`, `JoslCstDocument` |
|
|
282
|
+
| `@jarenjs/josl/gbnf` | `toGbnf`, `tomlToGbnf` |
|
|
283
|
+
| `@jarenjs/josl/stream` | `createStreamReader`, `parseJoslStream` |
|
|
284
|
+
| `@jarenjs/josl/stringify` | `stringifyJosl`, `stringifyToml`, `formatValue`, `formatSection` |
|
|
285
|
+
| `@jarenjs/josl/write` | `createStreamWriter`, `stringifyJoslChunks` |
|
|
286
|
+
| `@jarenjs/josl/jsonx` | `parseJsonx`, `stringifyJsonx` |
|
|
287
|
+
| `@jarenjs/josl/jsonx-stream` | `createJsonxStreamReader`, `parseJsonxStream` |
|
|
288
|
+
| `@jarenjs/josl/csv` | `parseCsv`, `parseCsvDocument`, `stringifyCsv`, `sniffCsvDialect` |
|
|
289
|
+
| `@jarenjs/josl/csv-stream` | `createCsvStreamReader`, `parseCsvStream`, `iterateCsvStream`, `createCsvStreamWriter` |
|
|
290
|
+
| `@jarenjs/josl/values` | `LocalDate`, `LocalTime`, `LocalDateTime` |
|
|
291
|
+
| `@jarenjs/josl/schemas/*` | schema artifacts (`jaren-josl-data.schema.json`) |
|
|
292
|
+
|
|
293
|
+
## CSV
|
|
294
|
+
|
|
295
|
+
The same machine shape as JOSL, applied to the format the world exports
|
|
296
|
+
by accident: one grammar path, wholesale and streaming, and a reader that
|
|
297
|
+
tells you what it had to fix.
|
|
298
|
+
|
|
299
|
+
```javascript
|
|
300
|
+
import { parseCsv, parseCsvDocument, stringifyCsv } from '@jarenjs/josl/csv';
|
|
301
|
+
import { createCsvStreamReader, iterateCsvStream } from '@jarenjs/josl/csv-stream';
|
|
302
|
+
|
|
303
|
+
parseCsv('a,b\n1,2'); // [['a','b'], ['1','2']]
|
|
304
|
+
parseCsv('a,b\n1,2', { headers: true }); // [{ a: '1', b: '2' }]
|
|
305
|
+
|
|
306
|
+
// rows as they complete, without ever holding the table
|
|
307
|
+
for await (const row of iterateCsvStream(response.body, { headers: true }))
|
|
308
|
+
await save(row);
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
**Strict by default.** Anything RFC 4180 forbids throws a
|
|
312
|
+
`CsvSyntaxError` carrying a `CSV1xxx` code, a line and a column — the
|
|
313
|
+
same machine-repairable error shape the JOSL parser uses.
|
|
314
|
+
|
|
315
|
+
**`repair: true` heals instead, and says so.** Every repair lands in a log
|
|
316
|
+
with the same code the strict error would have carried, so moving between
|
|
317
|
+
the modes never means re-learning the diagnosis:
|
|
318
|
+
|
|
319
|
+
```javascript
|
|
320
|
+
const doc = parseCsvDocument('a,b\n"he said "hi" ok",2\n', {
|
|
321
|
+
repair: true, headers: true,
|
|
322
|
+
});
|
|
323
|
+
doc.rows; // [{ a: 'he said "hi" ok', b: '2' }]
|
|
324
|
+
doc.repairs; // [{ code: 'CSV1003', line: 2, column: 10, message: … }, …]
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
| code | condition | how it is read |
|
|
328
|
+
| --- | --- | --- |
|
|
329
|
+
| `CSV1001` | a quoted field is never closed | close it at the end, keep the text |
|
|
330
|
+
| `CSV1002` | text after a closing quote | the field closed; absorb the stray text |
|
|
331
|
+
| `CSV1003` | an unescaped quote inside a quoted field | the quote is literal |
|
|
332
|
+
| `CSV1004` | a record shorter than the header | the columns stay **absent**, not empty |
|
|
333
|
+
| `CSV1005` | a record longer than the header | widen the header once |
|
|
334
|
+
| `CSV1006` | a bare carriage return | it ends the record (old-Mac endings) |
|
|
335
|
+
| `CSV1007` | a duplicate header name | suffix it (`a`, `a_2`) |
|
|
336
|
+
| `CSV1008` | an empty header name | name it (`column_3`) |
|
|
337
|
+
|
|
338
|
+
`CSV1002` and `CSV1003` are the same damage read two ways, and the reader
|
|
339
|
+
decides between them by looking for another quote before the next
|
|
340
|
+
delimiter. `"he said "hi" ok"` keeps its text; `"abc"junk,d` keeps its
|
|
341
|
+
**column count**, because a consumer indexes by column and a lost
|
|
342
|
+
boundary corrupts every field after it.
|
|
343
|
+
|
|
344
|
+
A short record leaves its missing columns absent rather than empty:
|
|
345
|
+
reading `undefined` says *this record did not carry the column*, where
|
|
346
|
+
`''` would claim it carried an empty one.
|
|
347
|
+
|
|
348
|
+
**Dialect sniffing.** `delimiter: 'auto'` scores each candidate by how
|
|
349
|
+
consistently it divides records — a real separator splits every record
|
|
350
|
+
the same way, a coincidental one splits them arbitrarily. Header
|
|
351
|
+
detection asks whether the first record looks unlike the rest, and
|
|
352
|
+
answers *no* when the table is text all the way down, because inventing a
|
|
353
|
+
header there would silently eat a data row.
|
|
354
|
+
|
|
355
|
+
**Typed values are the package's, not JSON's.** `typed: true` promotes an
|
|
356
|
+
integer past 2^53 to a **bigint** instead of rounding it, reads ISO dates
|
|
357
|
+
as the same `LocalDate`/`LocalDateTime` a JOSL document yields, and
|
|
358
|
+
leaves `007` a string — a zero-padded id does not survive becoming a
|
|
359
|
+
Number. A quoted cell is never coerced: the quotes are the author saying
|
|
360
|
+
this is text.
|
|
361
|
+
|
|
362
|
+
### CSV compliance & speed
|
|
363
|
+
|
|
364
|
+
`npm run benchmark:csv` scores the reader on
|
|
365
|
+
[csv-spectrum](https://www.npmjs.com/package/csv-spectrum), the de-facto
|
|
366
|
+
acceptance corpus, and on a scorecard of damaged documents.
|
|
367
|
+
|
|
368
|
+
<!--bm:csv.table-->
|
|
369
|
+
| engine | csv-spectrum | 10k×6 plain | 10k×3 quoted | 1k×50 wide |
|
|
370
|
+
| --- | --- | --- | --- | --- |
|
|
371
|
+
| **jaren** | **11/11** | 2.0 ms | 3.6 ms | 1.4 ms |
|
|
372
|
+
| udsv | 11/11 | **1.5 ms** | **3.0 ms** | **1.0 ms** |
|
|
373
|
+
| papaparse | 11/11 | 5.5 ms | 8.0 ms | 2.2 ms |
|
|
374
|
+
| csv-parse | 11/11 | 20.2 ms | 14.0 ms | 12.4 ms |
|
|
375
|
+
| d3-dsv | 11/11 | 4.3 ms | 6.3 ms | 2.6 ms |
|
|
376
|
+
| @vanillaes/csv | n/a | 7.5 ms | 9.6 ms | 6.1 ms |
|
|
377
|
+
<!--/bm-->
|
|
378
|
+
|
|
379
|
+
(The suite's twelfth fixture, `location_coordinates`, is excluded: its
|
|
380
|
+
expectation is a bare object where every other case is an array, its
|
|
381
|
+
phone number disagrees with its own CSV, and its degree sign is already
|
|
382
|
+
U+FFFD in the source bytes. No parser can satisfy it.)
|
|
383
|
+
|
|
384
|
+
**udsv is the honest loss: ~1.3× on the plain record stream, ~1.15× on
|
|
385
|
+
quoted, and ~2× when records become header-keyed objects.** It earns it —
|
|
386
|
+
udsv compiles a parser per schema with `new Function`, so each record
|
|
387
|
+
object is built with literal keys the engine can inline-cache, where this
|
|
388
|
+
reader assigns computed keys one by one. This package does not use
|
|
389
|
+
codegen, anywhere, by house rule: everything here runs under a strict
|
|
390
|
+
Content-Security-Policy, where runtime codegen is unavailable. That is
|
|
391
|
+
the same trade the schema validator and the query engine make, and it is
|
|
392
|
+
a trade rather than an excuse — against every parser that also avoids
|
|
393
|
+
codegen, jaren leads.
|
|
394
|
+
|
|
395
|
+
The plain-field scanner finds each cell end as the minimum of three
|
|
396
|
+
lazily-cached `indexOf` cursors (delimiter, LF, CR), so the source is
|
|
397
|
+
scanned by the engine's substring search rather than one character at a
|
|
398
|
+
time. Streaming is structurally a second walk — a chunk may stop
|
|
399
|
+
mid-field, so `feed` runs a side-effect-free cutter before the parser —
|
|
400
|
+
but the cutter cuts by the same substring search wherever no quote lies
|
|
401
|
+
ahead, and on the plain record stream the whole streaming read now costs
|
|
402
|
+
about 1.1× the wholesale path (2.1 ms vs 1.9 ms), level with udsv's
|
|
403
|
+
chunked reader.
|
|
404
|
+
|
|
405
|
+
Stringify leads everything that offers one: 3.6 ms against papaparse's
|
|
406
|
+
6.1 ms and d3-dsv's 4.1 ms.
|
|
407
|
+
|
|
408
|
+
## Generating JOSL with LLMs
|
|
409
|
+
|
|
410
|
+
JOSL is a *text* format, so "hand the grammar to a constrained decoder"
|
|
411
|
+
has two possible shapes, and they serve different providers:
|
|
412
|
+
|
|
413
|
+
1. **A JSON Schema over the data model** — the model emits JSON, and
|
|
414
|
+
`stringifyJosl` renders canonical JOSL text. This works with every
|
|
415
|
+
`json_schema`-capable provider today and composes directly with
|
|
416
|
+
`@jarenjs/ai`'s `createStructuredOutput`. It is what this package
|
|
417
|
+
ships: [`schemas/jaren-josl-data.schema.json`](./schemas/jaren-josl-data.schema.json)
|
|
418
|
+
describes the JSON-safe JOSL document (a root table of strings,
|
|
419
|
+
finite numbers, booleans, `null`, arrays and nested tables), and
|
|
420
|
+
`parseJosl(stringifyJosl(doc))` round-trips every document in it
|
|
421
|
+
exactly. JOSL's native date/time scalars parse to platform `Date`
|
|
422
|
+
values, which JSON cannot carry — represent them as strings and
|
|
423
|
+
parse downstream.
|
|
424
|
+
2. **A character-level grammar (GBNF class) over raw JOSL text** — for
|
|
425
|
+
engines that constrain token sampling directly (llama.cpp family).
|
|
426
|
+
`toGbnf()` emits it; `toGbnf({ mode: 'toml' })` drops the JOSL-only
|
|
427
|
+
value forms.
|
|
428
|
+
|
|
429
|
+
```js
|
|
430
|
+
import { toGbnf } from '@jarenjs/josl/gbnf';
|
|
431
|
+
|
|
432
|
+
await llama({ grammar: toGbnf(), prompt }); // emits JOSL text directly
|
|
433
|
+
```
|
|
434
|
+
|
|
435
|
+
A context-free grammar carries syntax only. Duplicate keys, a header
|
|
436
|
+
that reopens a table, `2026-02-30` — every rule that needs to
|
|
437
|
+
remember what the document already said, or to range-check a value
|
|
438
|
+
inside a well-formed token, stays `parseJosl`'s job. Constrained
|
|
439
|
+
sampling narrows the model to well-formed text; it does not make the
|
|
440
|
+
parser optional.
|
|
441
|
+
|
|
442
|
+
The grammar is checked both ways by `test/josl/gbnf.test.js`, which
|
|
443
|
+
reads it back and recognizes text with an Earley parser: it accepts
|
|
444
|
+
all 210 documents in the official toml-test valid corpus, rejects 400
|
|
445
|
+
of the 499 invalid ones (the remaining 99 fail on exactly the
|
|
446
|
+
semantic rules above), and 3000 seeded derivations across both modes all parse.
|
|
447
|
+
That last direction is the one that matters for sampling — a model
|
|
448
|
+
steered by this grammar cannot be walked into text the parser
|
|
449
|
+
rejects.
|
|
450
|
+
|
|
451
|
+
## Compliance & speed
|
|
452
|
+
|
|
453
|
+
Strict TOML mode is validated against the complete official
|
|
454
|
+
[toml-test](https://github.com/toml-lang/toml-test) 1.0.0 suite
|
|
455
|
+
(the git submodule at `benchmark/toml-test-suite/`) — every valid case with full
|
|
456
|
+
typed value verification (`npm run test:josl`), every invalid case
|
|
457
|
+
rejected. The only skips are eight byte-level UTF-8 encoding cases that
|
|
458
|
+
cannot be expressed once input is already a JS string.
|
|
459
|
+
|
|
460
|
+
`npm run benchmark:toml` runs the suite plus a parse/stringify profile
|
|
461
|
+
against `smol-toml`, `@iarna/toml` and `toml`. Representative run
|
|
462
|
+
(accept/reject compliance, 694 cases; Node 22):
|
|
463
|
+
|
|
464
|
+
| engine | compliance | parse, 1k-record doc | parse, small doc | parse, suite corpus |
|
|
465
|
+
| --- | --- | --- | --- | --- |
|
|
466
|
+
| **jaren** | **100.0%** | 7.4 ms | **0.014 ms** | 0.91 ms |
|
|
467
|
+
| smol-toml | 96.8% | **5.7 ms** | 0.021 ms | **0.80 ms** |
|
|
468
|
+
| @iarna/toml | 93.4% | 11.4 ms | 0.029 ms | 2.13 ms |
|
|
469
|
+
| toml | 98.6% | 38.4 ms | 0.080 ms | 4.83 ms |
|
|
470
|
+
|
|
471
|
+
jaren is the only engine at 100% and the only one that parses chunk
|
|
472
|
+
streams. Whole-document parsing walks the source once — the value
|
|
473
|
+
parsers already stop at the newlines TOML forbids a construct from
|
|
474
|
+
crossing, so with the whole text in hand the parser finds each logical
|
|
475
|
+
line's end itself and the cutter's separate pass is not needed. That is
|
|
476
|
+
worth ~1.45× over the two-pass version and puts jaren ahead on small
|
|
477
|
+
documents; on the 90 KB record stream smol-toml still leads, by ~1.3×
|
|
478
|
+
rather than the ~1.9× it led by before. Chunk feeding keeps the cutter,
|
|
479
|
+
because only a side-effect-free pre-pass can decide whether a line is
|
|
480
|
+
complete when a chunk may stop mid-token — and it is held to the same
|
|
481
|
+
694 cases, fed one character at a time.
|
|
482
|
+
|
|
483
|
+
Stringify leads: 2.5 ms against smol-toml's 2.7 ms and `@iarna/toml`'s
|
|
484
|
+
6.9 ms on the same document. The writer keeps its error path on a
|
|
485
|
+
mutable stack instead of allocating per key, emits each section header
|
|
486
|
+
from its parent's already-formatted prefix, and copies unescaped string
|
|
487
|
+
runs whole.
|
|
488
|
+
|
|
489
|
+
## Status
|
|
490
|
+
|
|
491
|
+
Published alongside the rest of the suite. Both constrained-decoding
|
|
492
|
+
twins ship — the JSON-Schema one over the data model and the GBNF one
|
|
493
|
+
over the text — along with the CST mode, progressive `text-partial`
|
|
494
|
+
events and single-walk whole-document parsing.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A parsed JOSL document that remembers its own text.
|
|
3
|
+
*
|
|
4
|
+
* Nodes are the document's logical lines in source order. Editing marks
|
|
5
|
+
* individual nodes dirty; every untouched node still prints as the exact
|
|
6
|
+
* bytes it was parsed from.
|
|
7
|
+
*/
|
|
8
|
+
export declare class JoslCstDocument {
|
|
9
|
+
source: string;
|
|
10
|
+
nodes: object[];
|
|
11
|
+
options: object;
|
|
12
|
+
bom: string;
|
|
13
|
+
_data: any;
|
|
14
|
+
_text: string;
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} source - The original document text
|
|
17
|
+
* @param {Array<object>} nodes - Logical-line nodes in source order
|
|
18
|
+
* @param {*} data - The parsed value model
|
|
19
|
+
* @param {object} options - Parse options, reused when re-reading edits
|
|
20
|
+
*/
|
|
21
|
+
constructor(source: string, nodes: Array<object>, data: any, options: object, bom?: string);
|
|
22
|
+
/**
|
|
23
|
+
* The document text, byte-identical to the input while unedited.
|
|
24
|
+
* @returns {string} JOSL source
|
|
25
|
+
*/
|
|
26
|
+
toString(): string;
|
|
27
|
+
/**
|
|
28
|
+
* The value model for the document as it currently reads. Recomputed
|
|
29
|
+
* from the text after an edit, so it can never drift from the bytes.
|
|
30
|
+
* @returns {*} The root table, or root array for [[]] documents
|
|
31
|
+
*/
|
|
32
|
+
toJSON(): any;
|
|
33
|
+
/**
|
|
34
|
+
* Read the value at a key path.
|
|
35
|
+
* @param {Array<string|number>} path - Absolute path of the pair
|
|
36
|
+
* @returns {*} The value, or undefined when the path holds no pair
|
|
37
|
+
*/
|
|
38
|
+
get(path: Array<string | number>): any;
|
|
39
|
+
/**
|
|
40
|
+
* Replace the value of an existing pair, or append a new pair to the
|
|
41
|
+
* section its path belongs to. Only the value's own bytes change, so the
|
|
42
|
+
* key's spelling, the spacing around `=` and any trailing comment stay
|
|
43
|
+
* exactly as written.
|
|
44
|
+
* @param {Array<string|number>} path - Absolute path of the pair
|
|
45
|
+
* @param {*} value - The new value
|
|
46
|
+
* @returns {this} The document, for chaining
|
|
47
|
+
*/
|
|
48
|
+
set(path: Array<string | number>, value: any): this;
|
|
49
|
+
/**
|
|
50
|
+
* Remove a pair, taking its whole line — including a trailing comment on
|
|
51
|
+
* that line — with it. Comments on their own lines are left alone.
|
|
52
|
+
* @param {Array<string|number>} path - Absolute path of the pair
|
|
53
|
+
* @returns {boolean} True when a pair was removed
|
|
54
|
+
*/
|
|
55
|
+
delete(path: Array<string | number>): boolean;
|
|
56
|
+
invalidate(): this;
|
|
57
|
+
insertPair(path: any, value: any, rendered: any): this;
|
|
58
|
+
firstHeaderIndex(): number;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Parse a document into a CST that preserves its exact text.
|
|
62
|
+
* @param {string} text - JOSL source text
|
|
63
|
+
* @param {object} [options] - Reader options
|
|
64
|
+
* @param {'josl'|'toml'} [options.mode] - 'toml' rejects JOSL extensions
|
|
65
|
+
* @returns {JoslCstDocument} The document
|
|
66
|
+
* @throws {import('./errors.js').JoslSyntaxError} On invalid input
|
|
67
|
+
*/
|
|
68
|
+
export declare function parseJoslCst(text: string, options?: {
|
|
69
|
+
mode?: 'josl' | 'toml';
|
|
70
|
+
}): JoslCstDocument;
|
|
71
|
+
/**
|
|
72
|
+
* Parse a CST in strict TOML 1.0 mode.
|
|
73
|
+
* @param {string} text - TOML source text
|
|
74
|
+
* @param {object} [options] - Reader options minus `mode`
|
|
75
|
+
* @returns {JoslCstDocument} The document
|
|
76
|
+
*/
|
|
77
|
+
export declare function parseTomlCst(text: string, options?: object): JoslCstDocument;
|
|
78
|
+
export { JoslSyntaxError } from './errors.js';
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The conditions the reader can diagnose. Each is a strict-mode error
|
|
3
|
+
* code and a repair-mode log code; the wording is the same either way.
|
|
4
|
+
*/
|
|
5
|
+
export declare const CSV_CODES: {
|
|
6
|
+
CSV1001: string;
|
|
7
|
+
CSV1002: string;
|
|
8
|
+
CSV1003: string;
|
|
9
|
+
CSV1004: string;
|
|
10
|
+
CSV1005: string;
|
|
11
|
+
CSV1006: string;
|
|
12
|
+
CSV1007: string;
|
|
13
|
+
CSV1008: string;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Coerce one cell to a JOSL value, or return the string unchanged.
|
|
17
|
+
*
|
|
18
|
+
* The value model is the package's, not JSON's: an integer too large for
|
|
19
|
+
* a Number becomes a BigInt rather than silently losing digits, and an
|
|
20
|
+
* unambiguous ISO-8601 date becomes the same `LocalDate`/`LocalDateTime`
|
|
21
|
+
* a JOSL document would produce. Anything the grammar does not match
|
|
22
|
+
* exactly stays a string — CSV has no types, so a reader must not guess
|
|
23
|
+
* beyond what the text says outright.
|
|
24
|
+
* @param {string} s - The raw cell text
|
|
25
|
+
* @returns {*} The coerced value, or `s` unchanged
|
|
26
|
+
*/
|
|
27
|
+
export declare function coerceCsvValue(s: string): any;
|
|
28
|
+
export declare class CsvMachine {
|
|
29
|
+
#private;
|
|
30
|
+
delimiter: any;
|
|
31
|
+
delimChar: string;
|
|
32
|
+
quote: any;
|
|
33
|
+
quoteChar: string;
|
|
34
|
+
comment: any;
|
|
35
|
+
repair: boolean;
|
|
36
|
+
trim: boolean;
|
|
37
|
+
typed: boolean;
|
|
38
|
+
emptyAsNull: boolean;
|
|
39
|
+
skipEmptyLines: boolean;
|
|
40
|
+
onEvent: any;
|
|
41
|
+
onRepair: any;
|
|
42
|
+
wantHeader: boolean;
|
|
43
|
+
headerFields: any;
|
|
44
|
+
objectRows: boolean;
|
|
45
|
+
plainCells: boolean;
|
|
46
|
+
protoSafe: any;
|
|
47
|
+
buf: string;
|
|
48
|
+
scanPos: number;
|
|
49
|
+
scanState: number;
|
|
50
|
+
started: boolean;
|
|
51
|
+
ended: boolean;
|
|
52
|
+
nextDelim: number;
|
|
53
|
+
nextLf: number;
|
|
54
|
+
nextCr: number;
|
|
55
|
+
outRows: any[];
|
|
56
|
+
repairLog: any[];
|
|
57
|
+
cells: any[];
|
|
58
|
+
recordIndex: number;
|
|
59
|
+
dropRecord: boolean;
|
|
60
|
+
line: number;
|
|
61
|
+
recordOrigin: number;
|
|
62
|
+
/**
|
|
63
|
+
* @param {object} [options] - Reader options; see `parseCsv`
|
|
64
|
+
*/
|
|
65
|
+
constructor(options?: object);
|
|
66
|
+
/**
|
|
67
|
+
* Feed the next chunk of source text; chunks may split any field.
|
|
68
|
+
* @param {string} chunk - Next piece of the document
|
|
69
|
+
* @returns {this} The machine, for chaining
|
|
70
|
+
*/
|
|
71
|
+
feed(chunk: string): this;
|
|
72
|
+
/**
|
|
73
|
+
* Finish the document, flushing any pending record.
|
|
74
|
+
* @returns {Array} The completed rows
|
|
75
|
+
*/
|
|
76
|
+
end(): any[];
|
|
77
|
+
/**
|
|
78
|
+
* Parse a complete document in one pass. `parseRecord` finds each
|
|
79
|
+
* record's end as it goes, so with the whole text in hand the cutter's
|
|
80
|
+
* separate pass is not needed; `feed`/`end` keep it because a chunk can
|
|
81
|
+
* stop mid-field, where only a side-effect-free pre-pass can decide
|
|
82
|
+
* whether a record is complete.
|
|
83
|
+
* @param {string} text - The entire document
|
|
84
|
+
* @returns {Array} The completed rows
|
|
85
|
+
*/
|
|
86
|
+
parseAll(text: string): any[];
|
|
87
|
+
/** @returns {Array} The rows read so far. */
|
|
88
|
+
rows(): any[];
|
|
89
|
+
/** @returns {string[]|null} The header names, once known. */
|
|
90
|
+
fields(): string[] | null;
|
|
91
|
+
/** @returns {object[]} The repair log, in document order. */
|
|
92
|
+
repairs(): object[];
|
|
93
|
+
heal(code: any, line: any, column: any, detail?: undefined): void;
|
|
94
|
+
scan(): void;
|
|
95
|
+
compact(start: any): void;
|
|
96
|
+
readSpan(text: any, pos: any, end: any): any;
|
|
97
|
+
parseRecord(text: any, pos: any, end: any, cells: any): any;
|
|
98
|
+
parsePlain(text: any, pos: any, end: any, cells: any): any;
|
|
99
|
+
parseQuoted(text: any, pos: any, end: any, cells: any): any;
|
|
100
|
+
finish(cell: any): any;
|
|
101
|
+
finishQuoted(cell: any): any;
|
|
102
|
+
emitRecord(cells: any): void;
|
|
103
|
+
buildObject(values: any): {};
|
|
104
|
+
}
|