@fraczak/k 6.10.1 → 6.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/DOCS/BIG_PICTURE.md +174 -100
- package/DOCS/CODECS.md +74 -29
- package/DOCS/CODEC_ARCHITECTURE.md +87 -31
- package/DOCS/CONVERGENCE.md +42 -11
- package/DOCS/DICTIONARY.md +69 -2
- package/DOCS/FILE_FORMATS.md +385 -0
- package/DOCS/KIR_V1.md +53 -42
- package/DOCS/KVM_EXECUTION_MODEL.md +189 -147
- package/DOCS/OBJECT_FILE_AND_PATTERN.md +35 -17
- package/DOCS/PATTERNS.md +10 -6
- package/DOCS/REPL.md +100 -60
- package/DOCS/TEXTUAL_VALUES.md +24 -19
- package/DOCS/TYPE_DERIVATION.md +68 -22
- package/README.md +92 -58
- package/codecs/README.md +78 -67
- package/codecs/runtime/show-value.mjs +68 -16
- package/objects/README.md +57 -37
- package/package.json +1 -1
- package/tests/README.md +28 -17
- package/tests/test-repl-html.mjs +19 -0
package/README.md
CHANGED
|
@@ -3,14 +3,15 @@
|
|
|
3
3
|
**k is a small language for typed data transformations.**
|
|
4
4
|
|
|
5
5
|
It describes data as algebraic shapes, programs as first-order partial
|
|
6
|
-
functions, and runtime values as a binary `pattern + value` stream
|
|
7
|
-
parsed, transformed, printed, compiled, and inspected.
|
|
6
|
+
functions, and runtime values as a binary `pattern + value` stream
|
|
7
|
+
that can be parsed, transformed, printed, compiled, and inspected.
|
|
8
8
|
|
|
9
|
-
Data descriptions and transformations share one syntax, so a k file can
|
|
10
|
-
both the shapes of values and the relations that move between them.
|
|
9
|
+
Data descriptions and transformations share one syntax, so a k file can
|
|
10
|
+
define both the shapes of values and the relations that move between them.
|
|
11
11
|
|
|
12
|
-
k is experimental, but it already has a working parser, type-derivation
|
|
13
|
-
REPL, binary codec pipeline, object/library format, Node.js API,
|
|
12
|
+
k is experimental, but it already has a working parser, type-derivation
|
|
13
|
+
engine, REPL, binary codec pipeline, object/library format, Node.js API,
|
|
14
|
+
and test suite.
|
|
14
15
|
|
|
15
16
|
## Why k?
|
|
16
17
|
|
|
@@ -21,9 +22,10 @@ Many systems need to answer the same questions:
|
|
|
21
22
|
- What does a program accept, produce, or reject?
|
|
22
23
|
- Can this transformation be serialized, tested, reused, or inspected?
|
|
23
24
|
|
|
24
|
-
k explores a compact answer: define algebraic data shapes, compose
|
|
25
|
-
transformations over them, derive input/output patterns
|
|
26
|
-
values across process boundaries with a canonical
|
|
25
|
+
k explores a compact answer: define algebraic data shapes, compose
|
|
26
|
+
partial transformations over them, derive input/output patterns
|
|
27
|
+
automatically, and move values across process boundaries with a canonical
|
|
28
|
+
binary representation.
|
|
27
29
|
|
|
28
30
|
That makes k interesting as a foundation for:
|
|
29
31
|
|
|
@@ -35,8 +37,8 @@ That makes k interesting as a foundation for:
|
|
|
35
37
|
|
|
36
38
|
## A Small Example
|
|
37
39
|
|
|
38
|
-
k can define recursive data and transformations in the same file. Peano
|
|
39
|
-
numbers are either `0` or one more than another natural number:
|
|
40
|
+
k can define recursive data and transformations in the same file. Peano
|
|
41
|
+
natural numbers are either `0` or one more than another natural number:
|
|
40
42
|
|
|
41
43
|
```k
|
|
42
44
|
$ nat = < {} 0, nat +1 >;
|
|
@@ -49,8 +51,9 @@ add = $ { nat x, nat y } <
|
|
|
49
51
|
>;
|
|
50
52
|
```
|
|
51
53
|
|
|
52
|
-
`add` takes a product `{ x, y }`. If `x` has a `+1`, it moves that
|
|
53
|
-
from `x` to `y` and recurses. When that no longer applies, it
|
|
54
|
+
`add` takes a product `{ x, y }`. If `x` has a `+1`, it moves that
|
|
55
|
+
successor from `x` to `y` and recurses. When that no longer applies, it
|
|
56
|
+
returns `y`.
|
|
54
57
|
|
|
55
58
|
In the REPL:
|
|
56
59
|
|
|
@@ -59,8 +62,8 @@ In the REPL:
|
|
|
59
62
|
{}|0|+1|+1|+1 ?<X0 +1, {} 0>=X0
|
|
60
63
|
```
|
|
61
64
|
|
|
62
|
-
That evaluates `2 + 1` to `3`: a value starts at `0`, and each `+1` tag
|
|
63
|
-
one successor. The REPL also prints the inferred value envelope.
|
|
65
|
+
That evaluates `2 + 1` to `3`: a value starts at `0`, and each `+1` tag
|
|
66
|
+
adds one successor. The REPL also prints the inferred value envelope.
|
|
64
67
|
|
|
65
68
|
## Try It
|
|
66
69
|
|
|
@@ -69,7 +72,9 @@ one successor. The REPL also prints the inferred value envelope.
|
|
|
69
72
|
Try k directly in your browser without installing anything:
|
|
70
73
|
**[https://fraczak.github.io/k/](https://fraczak.github.io/k/)**
|
|
71
74
|
|
|
72
|
-
The Web REPL is self-contained, powered by an in-process WebAssembly
|
|
75
|
+
The Web REPL is self-contained, powered by an in-process WebAssembly
|
|
76
|
+
compiler and execution engine with built-in codecs, autocompletion,
|
|
77
|
+
interactive input, and standard libraries.
|
|
73
78
|
|
|
74
79
|
### From Source
|
|
75
80
|
|
|
@@ -121,8 +126,8 @@ $ byte = { bit 0, bit 1, bit 2, bit 3, bit 4, bit 5, bit 6, bit 7 };
|
|
|
121
126
|
$ bits = < {} _, bits 0, bits 1 >;
|
|
122
127
|
```
|
|
123
128
|
|
|
124
|
-
There are no built-in primitive values. The empty product `{}` is the
|
|
125
|
-
in a non-recursive definition.
|
|
129
|
+
There are no built-in primitive values. The empty product `{}` is the
|
|
130
|
+
only leaf in a non-recursive definition.
|
|
126
131
|
|
|
127
132
|
**Composable partial functions**
|
|
128
133
|
|
|
@@ -142,8 +147,9 @@ Core expressions are deliberately small:
|
|
|
142
147
|
|
|
143
148
|
**Derived input/output patterns**
|
|
144
149
|
|
|
145
|
-
k derives structural constraints for expressions. Those patterns become
|
|
146
|
-
for diagnostics, REPL output, binary encoding, and object
|
|
150
|
+
k derives structural constraints for expressions. Those patterns become
|
|
151
|
+
useful for diagnostics, REPL output, binary encoding, and object
|
|
152
|
+
metadata.
|
|
147
153
|
|
|
148
154
|
**Binary-friendly runtime values**
|
|
149
155
|
|
|
@@ -163,15 +169,27 @@ k-print # binary pattern+value stream -> textual value
|
|
|
163
169
|
|
|
164
170
|
**Inspectable objects and libraries**
|
|
165
171
|
|
|
166
|
-
k programs can be compiled
|
|
172
|
+
`k` programs can be authored, compiled, and linked across four main
|
|
173
|
+
file formats:
|
|
167
174
|
|
|
168
|
-
- `.
|
|
169
|
-
- `.
|
|
170
|
-
|
|
175
|
+
- `.k`: human-readable source code defining algebraic types and relations.
|
|
176
|
+
- `.klib`: ahead-of-time (AOT) type-checked library objects (`main: null`)
|
|
177
|
+
storing canonical relation definitions, type codes, and local aliases.
|
|
178
|
+
- `.ko`: ahead-of-time (AOT) type-checked executable containers (`KOBJ\n`
|
|
179
|
+
magic header) with entrypoint `main: "__main__"`, skipping parsing and
|
|
180
|
+
type derivation at runtime.
|
|
181
|
+
- `.kvm`: polymorphic register-IR template artifacts (`layer: "KVM-P"`)
|
|
182
|
+
carrying principal pattern graphs for instant runtime specialization
|
|
183
|
+
(`specializeKVM`).
|
|
171
184
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
185
|
+
In `k`, all relations and types are identified canonically by immutable
|
|
186
|
+
content-addressed hashes (`@hash`). There is no global symbol namespace;
|
|
187
|
+
friendly names (such as `plus`, `int`, or `5`) are local aliases stored
|
|
188
|
+
in metadata. Similarly, there are no primitive numbers: numerical
|
|
189
|
+
identifiers like `5` or `10` are nullary relations mapping `${} -> $bits`.
|
|
190
|
+
|
|
191
|
+
For a detailed guide to all four formats, commands, and workflows, see
|
|
192
|
+
[DOCS/FILE_FORMATS.md](DOCS/FILE_FORMATS.md).
|
|
175
193
|
|
|
176
194
|
## CLI Tour
|
|
177
195
|
|
|
@@ -215,20 +233,22 @@ artifact for inspection and execution experiments.
|
|
|
215
233
|
| `k-llvm-build` / `k-llvm-compile` / `k-llvm-run` | Compile and run via LLVM backend |
|
|
216
234
|
| `k-arm64` / `k-arm64-compile` / `k-arm64-run` | Compile and run via Linux ARM64 native backend |
|
|
217
235
|
|
|
218
|
-
Installed binary names are `k-` plus the source basename without
|
|
219
|
-
for `k.mjs` itself. Source names that already include
|
|
220
|
-
`codecs/k-parse.mjs`, keep that name.
|
|
236
|
+
Installed binary names are `k-` plus the source basename without
|
|
237
|
+
`.mjs`, except for `k.mjs` itself. Source names that already include
|
|
238
|
+
`k-`, such as `codecs/k-parse.mjs`, keep that name.
|
|
221
239
|
|
|
222
240
|
## Backends
|
|
223
241
|
|
|
224
242
|
Backends live under [`backends/`](backends/) as npm workspaces:
|
|
225
243
|
|
|
226
|
-
- [`backends/wasm`](backends/wasm/) lowers typed k programs through kVM
|
|
227
|
-
WebAssembly artifacts. Powers both the CLI REPL (`wasm-in-process`)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
244
|
+
- [`backends/wasm`](backends/wasm/) lowers typed k programs through kVM
|
|
245
|
+
into WebAssembly artifacts. Powers both the CLI REPL (`wasm-in-process`)
|
|
246
|
+
and the standalone Web REPL.
|
|
247
|
+
- [`backends/llvm`](backends/llvm/) lowers polymorphic kVM programs into
|
|
248
|
+
LLVM IR and compiled native test executables.
|
|
249
|
+
- [`backends/arm64`](backends/arm64/) lowers polymorphic kVM programs
|
|
250
|
+
directly into standalone Linux ARM64 native executables without external
|
|
251
|
+
compiler dependencies.
|
|
232
252
|
|
|
233
253
|
All backends integrate with the compiler and binary codecs through
|
|
234
254
|
[`backend-api.mjs`](backend-api.mjs).
|
|
@@ -255,8 +275,8 @@ console.log(annotated.compileStats);
|
|
|
255
275
|
|
|
256
276
|
## Project Status
|
|
257
277
|
|
|
258
|
-
k is usable as an experimental language and toolkit, not a stable
|
|
259
|
-
platform yet.
|
|
278
|
+
k is usable as an experimental language and toolkit, not a stable
|
|
279
|
+
production platform yet.
|
|
260
280
|
|
|
261
281
|
Working today:
|
|
262
282
|
|
|
@@ -266,7 +286,8 @@ Working today:
|
|
|
266
286
|
- binary pattern+value codec
|
|
267
287
|
- object and library files
|
|
268
288
|
- Node.js API
|
|
269
|
-
- regression tests for runtime, codecs, objects, hashes, and type
|
|
289
|
+
- regression tests for runtime, codecs, objects, hashes, and type
|
|
290
|
+
derivation
|
|
270
291
|
|
|
271
292
|
Still evolving:
|
|
272
293
|
|
|
@@ -281,20 +302,24 @@ Still evolving:
|
|
|
281
302
|
|
|
282
303
|
k is small enough to study, but there are several useful directions:
|
|
283
304
|
|
|
284
|
-
- examples: schema transformations, codecs, protocol examples, teaching
|
|
305
|
+
- examples: schema transformations, codecs, protocol examples, teaching
|
|
306
|
+
tasks
|
|
285
307
|
- documentation: tutorials, diagrams, and clearer language walkthroughs
|
|
286
|
-
- tooling: formatter, editor integration, better diagnostics, REPL
|
|
308
|
+
- tooling: formatter, editor integration, better diagnostics, REPL
|
|
309
|
+
ergonomics
|
|
287
310
|
- compiler work: optimization, object inspection, backend experiments
|
|
288
311
|
- theory: normalization, equivalence, convergence, and pattern derivation
|
|
289
|
-
- applications: hardware modeling, asynchronous/synchronous
|
|
312
|
+
- applications: hardware modeling, asynchronous/synchronous pipelines,
|
|
290
313
|
schema repositories, and data migration tooling
|
|
291
314
|
|
|
292
|
-
If you are interested in languages, compilers, data modeling, formal
|
|
293
|
-
serialization, or teaching tools, there is room to shape the
|
|
315
|
+
If you are interested in languages, compilers, data modeling, formal
|
|
316
|
+
methods, serialization, or teaching tools, there is room to shape the
|
|
317
|
+
project.
|
|
294
318
|
|
|
295
319
|
## Examples
|
|
296
320
|
|
|
297
|
-
The [`Examples/`](Examples/) directory contains language demonstrations
|
|
321
|
+
The [`Examples/`](Examples/) directory contains language demonstrations
|
|
322
|
+
and standard libraries:
|
|
298
323
|
|
|
299
324
|
| File | Contents |
|
|
300
325
|
| --- | --- |
|
|
@@ -307,13 +332,14 @@ The [`Examples/`](Examples/) directory contains language demonstrations and stan
|
|
|
307
332
|
| `bnat.k` | Binary natural numbers |
|
|
308
333
|
| `buda.k` | Compact relational data transformations |
|
|
309
334
|
|
|
310
|
-
`list.k` is also useful as a focused demonstration of filters and
|
|
335
|
+
`list.k` is also useful as a focused demonstration of filters and
|
|
336
|
+
patterns.
|
|
311
337
|
|
|
312
|
-
`ieee.k` is a complete IEEE-754 binary64 model built from bit-level
|
|
313
|
-
upward with hierarchical significand adders, including comparison
|
|
314
|
-
floating-point `add`, `sub`, `mul`, and `div` relations. Those public
|
|
315
|
-
return `{ result, flags }`; compose with `.result` when only the
|
|
316
|
-
value is needed.
|
|
338
|
+
`ieee.k` is a complete IEEE-754 binary64 model built from bit-level
|
|
339
|
+
types upward with hierarchical significand adders, including comparison
|
|
340
|
+
and floating-point `add`, `sub`, `mul`, and `div` relations. Those public
|
|
341
|
+
aliases return `{ result, flags }`; compose with `.result` when only the
|
|
342
|
+
floating-point value is needed.
|
|
317
343
|
|
|
318
344
|
## Development
|
|
319
345
|
|
|
@@ -331,8 +357,9 @@ npm run perf:int # run integer arithmetic benchmark across backends
|
|
|
331
357
|
npm run perf:ieee # run IEEE-754 arithmetic benchmark across backends
|
|
332
358
|
```
|
|
333
359
|
|
|
334
|
-
The test runner prints each test before execution and reports its
|
|
335
|
-
afterward. It stops immediately when a test fails.
|
|
360
|
+
The test runner prints each test before execution and reports its
|
|
361
|
+
elapsed time afterward. It stops immediately when a test fails.
|
|
362
|
+
The suite covers:
|
|
336
363
|
|
|
337
364
|
- core runtime and parser behavior
|
|
338
365
|
- type derivation cases in `tests/code-derivation/`
|
|
@@ -345,14 +372,21 @@ afterward. It stops immediately when a test fails. The suite covers:
|
|
|
345
372
|
|
|
346
373
|
## Further Reading
|
|
347
374
|
|
|
375
|
+
- [DOCS/FILE_FORMATS.md](DOCS/FILE_FORMATS.md) - guide to
|
|
376
|
+
.k, .ko, .klib, and .kvm file formats and workflows
|
|
348
377
|
- [DOCS/DICTIONARY.md](DOCS/DICTIONARY.md) - concept names and terminology
|
|
349
378
|
- [DOCS/REPL.md](DOCS/REPL.md) - interactive interpreter and codec details
|
|
350
|
-
- [DOCS/WEB_REPL_REQUIREMENTS.md](DOCS/WEB_REPL_REQUIREMENTS.md) -
|
|
351
|
-
|
|
352
|
-
- [DOCS/
|
|
379
|
+
- [DOCS/WEB_REPL_REQUIREMENTS.md](DOCS/WEB_REPL_REQUIREMENTS.md) -
|
|
380
|
+
Web REPL architecture and requirements
|
|
381
|
+
- [DOCS/KVM_EXECUTION_MODEL.md](DOCS/KVM_EXECUTION_MODEL.md) -
|
|
382
|
+
kVM execution model and bytecode design
|
|
383
|
+
- [DOCS/TEXTUAL_VALUES.md](DOCS/TEXTUAL_VALUES.md) -
|
|
384
|
+
textual boundary notation
|
|
353
385
|
- [DOCS/PATTERNS.md](DOCS/PATTERNS.md) - pattern representation
|
|
354
|
-
- [DOCS/OBJECT_FILE_AND_PATTERN.md](DOCS/OBJECT_FILE_AND_PATTERN.md) -
|
|
355
|
-
|
|
386
|
+
- [DOCS/OBJECT_FILE_AND_PATTERN.md](DOCS/OBJECT_FILE_AND_PATTERN.md) -
|
|
387
|
+
object format and pattern encoding
|
|
388
|
+
- [DOCS/PROFILING_AND_TRACING.md](DOCS/PROFILING_AND_TRACING.md) -
|
|
389
|
+
execution phase tracing and function call profiling across backends
|
|
356
390
|
- [DOCS/CODECS.md](DOCS/CODECS.md) - writing external codecs
|
|
357
391
|
- [codecs/README.md](codecs/README.md) - binary codec internals
|
|
358
392
|
- [objects/README.md](objects/README.md) - object/library tools
|
package/codecs/README.md
CHANGED
|
@@ -1,35 +1,36 @@
|
|
|
1
1
|
# K Codecs
|
|
2
2
|
|
|
3
|
-
This directory is
|
|
3
|
+
This directory is centered on the codec design:
|
|
4
4
|
|
|
5
5
|
```text
|
|
6
6
|
abstract pattern graph + prefix-free tree encoding
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
-
The default concrete format is now the binary encoding of a `$pattern`
|
|
10
|
-
using the `$pattern` type from [`../core.k`](../core.k),
|
|
11
|
-
the binary encoding of the value under that
|
|
9
|
+
The default concrete format is now the binary encoding of a `$pattern`
|
|
10
|
+
value, using the `$pattern` type from [`../core.k`](../core.k),
|
|
11
|
+
immediately followed by the binary encoding of the value under that
|
|
12
|
+
decoded pattern.
|
|
12
13
|
|
|
13
|
-
`core.k` is not loaded automatically as a library when user programs
|
|
14
|
-
codec only depends on the canonical `$pattern` definition from
|
|
15
|
-
schema for the leading pattern value.
|
|
14
|
+
`core.k` is not loaded automatically as a library when user programs
|
|
15
|
+
run. The codec only depends on the canonical `$pattern` definition from
|
|
16
|
+
that file as the schema for the leading pattern value.
|
|
16
17
|
|
|
17
18
|
There is no separate JSON container format.
|
|
18
19
|
|
|
19
20
|
## Objectives
|
|
20
21
|
|
|
21
|
-
The
|
|
22
|
+
The design is guided by two hard requirements:
|
|
22
23
|
|
|
23
24
|
1. compactness close to the structural lower bound,
|
|
24
25
|
2. efficient `k` projections, especially `.field` and `/tag`.
|
|
25
26
|
|
|
26
|
-
The current target architecture is the pattern graph plus prefix-free
|
|
27
|
-
stream described here and in
|
|
27
|
+
The current target architecture is the pattern graph plus prefix-free
|
|
28
|
+
value-tree stream described here and in
|
|
28
29
|
[`POLYMORPHIC_BINARY_FORMAT.md`](./POLYMORPHIC_BINARY_FORMAT.md).
|
|
29
30
|
|
|
30
31
|
## Core Idea
|
|
31
32
|
|
|
32
|
-
The semantic object serialized by the
|
|
33
|
+
The semantic object serialized by the codec is:
|
|
33
34
|
|
|
34
35
|
```text
|
|
35
36
|
(P, v)
|
|
@@ -40,10 +41,11 @@ where:
|
|
|
40
41
|
- `P` is an abstract rooted pattern graph,
|
|
41
42
|
- `v` is a value tree compatible with `P`.
|
|
42
43
|
|
|
43
|
-
The same pair is
|
|
44
|
-
attaches `P` to the resulting `Value` as `value.pattern`; evaluation
|
|
45
|
-
that pattern through projections and constructors; encoding
|
|
46
|
-
carried pattern unless an explicit pattern is
|
|
44
|
+
The same pair is the in-memory runtime model as well. Decoding a stream
|
|
45
|
+
attaches `P` to the resulting `Value` as `value.pattern`; evaluation
|
|
46
|
+
propagates that pattern through projections and constructors; encoding
|
|
47
|
+
a `Value` uses its carried pattern unless an explicit pattern is
|
|
48
|
+
supplied.
|
|
47
49
|
|
|
48
50
|
The base value encoding is:
|
|
49
51
|
|
|
@@ -52,9 +54,9 @@ The base value encoding is:
|
|
|
52
54
|
- interpreted relative to the pattern graph,
|
|
53
55
|
- free of repeated field names and tag names in the value payload.
|
|
54
56
|
|
|
55
|
-
The base codec does **not** perform DAG compression. If sharing is
|
|
56
|
-
should happen as a higher-level transform, not inside the
|
|
57
|
-
format.
|
|
57
|
+
The base codec does **not** perform DAG compression. If sharing is
|
|
58
|
+
desired, it should happen as a higher-level transform, not inside the
|
|
59
|
+
primitive value format.
|
|
58
60
|
|
|
59
61
|
## Wire Format
|
|
60
62
|
|
|
@@ -64,9 +66,9 @@ The default transport format is:
|
|
|
64
66
|
encode($pattern_value : $pattern) encode(value : decoded_pattern)
|
|
65
67
|
```
|
|
66
68
|
|
|
67
|
-
`$pattern_value` is a normal k value whose shape is defined in `core.k`.
|
|
68
|
-
that value is decoded, it becomes the pattern used to decode the
|
|
69
|
-
value payload.
|
|
69
|
+
`$pattern_value` is a normal k value whose shape is defined in `core.k`.
|
|
70
|
+
After that value is decoded, it becomes the pattern used to decode the
|
|
71
|
+
remaining value payload.
|
|
70
72
|
|
|
71
73
|
The runtime boundary is therefore:
|
|
72
74
|
|
|
@@ -74,25 +76,28 @@ The runtime boundary is therefore:
|
|
|
74
76
|
binary pattern+value stream <-> Value(pattern, tree)
|
|
75
77
|
```
|
|
76
78
|
|
|
77
|
-
At the command-line boundary, `k-parse` and `k-print` use a JSON-like
|
|
78
|
-
notation for k values. That notation is documented in
|
|
79
|
-
[`../DOCS/TEXTUAL_VALUES.md`](../DOCS/TEXTUAL_VALUES.md). It is a
|
|
80
|
-
notation for k values, not a separate JSON container format.
|
|
79
|
+
At the command-line boundary, `k-parse` and `k-print` use a JSON-like
|
|
80
|
+
textual notation for k values. That notation is documented in
|
|
81
|
+
[`../DOCS/TEXTUAL_VALUES.md`](../DOCS/TEXTUAL_VALUES.md). It is a
|
|
82
|
+
readable notation for k values, not a separate JSON container format.
|
|
81
83
|
|
|
82
84
|
For the codec architecture and specification, see
|
|
83
85
|
[`../DOCS/CODEC_ARCHITECTURE.md`](../DOCS/CODEC_ARCHITECTURE.md).
|
|
84
|
-
For the contributor-facing guide to writing a new external codec module,
|
|
85
|
-
[`../DOCS/CODECS.md`](../DOCS/CODECS.md).
|
|
86
|
+
For the contributor-facing guide to writing a new external codec module,
|
|
87
|
+
see [`../DOCS/CODECS.md`](../DOCS/CODECS.md).
|
|
88
|
+
For the broader view of all artifacts and compilation targets, see
|
|
89
|
+
[`../DOCS/FILE_FORMATS.md`](../DOCS/FILE_FORMATS.md).
|
|
86
90
|
|
|
87
|
-
`k.mjs` is only the command-line adapter for that boundary. The
|
|
88
|
-
runtime sees and preserves the pattern on the `Value`
|
|
91
|
+
`k.mjs` is only the command-line adapter for that boundary. The
|
|
92
|
+
operational runtime sees and preserves the pattern on the `Value`
|
|
93
|
+
itself.
|
|
89
94
|
|
|
90
95
|
## Command-Line Tools
|
|
91
96
|
|
|
92
|
-
Installed codec binaries use the `k-` prefix followed by the source
|
|
93
|
-
without `.mjs`. Source names that already include `k-`, such
|
|
94
|
-
`k-parse.mjs`, keep that name. Every installed codec binary supports
|
|
95
|
-
`--help`.
|
|
97
|
+
Installed codec binaries use the `k-` prefix followed by the source
|
|
98
|
+
basename without `.mjs`. Source names that already include `k-`, such
|
|
99
|
+
as `k-parse.mjs`, keep that name. Every installed codec binary supports
|
|
100
|
+
`-h` and `--help`.
|
|
96
101
|
|
|
97
102
|
| Binary | Source | Purpose |
|
|
98
103
|
| --- | --- | --- |
|
|
@@ -108,8 +113,8 @@ without `.mjs`. Source names that already include `k-`, such as
|
|
|
108
113
|
|
|
109
114
|
## Pattern Graph Representation
|
|
110
115
|
|
|
111
|
-
For documentation and tests, a pattern graph may be shown as a
|
|
112
|
-
style vector of nodes:
|
|
116
|
+
For documentation and tests, a pattern graph may be shown as a
|
|
117
|
+
property-list style vector of nodes:
|
|
113
118
|
|
|
114
119
|
```text
|
|
115
120
|
[kind, edges]
|
|
@@ -145,22 +150,23 @@ Canonical rules:
|
|
|
145
150
|
- edge labels are unique within a node,
|
|
146
151
|
- `"any"` must have no outgoing edges.
|
|
147
152
|
|
|
148
|
-
This JSON-like graph is only a readable notation. The wire
|
|
149
|
-
the ordinary k `$pattern` value from `core.k`.
|
|
153
|
+
This JSON-like graph is only a readable notation. The wire
|
|
154
|
+
representation is the ordinary k `$pattern` value from `core.k`.
|
|
150
155
|
|
|
151
|
-
For pattern export, see [`../DOCS/PATTERNS.md`](../DOCS/PATTERNS.md) and
|
|
152
|
-
repository helper `patterns/from-k.mjs`.
|
|
156
|
+
For pattern export, see [`../DOCS/PATTERNS.md`](../DOCS/PATTERNS.md) and
|
|
157
|
+
the repository helper `patterns/from-k.mjs`.
|
|
153
158
|
|
|
154
159
|
## Witness-Derived Patterns
|
|
155
160
|
|
|
156
|
-
When `k-parse` is used without an explicit input pattern or type, the
|
|
157
|
-
pattern is derived from the parsed value tree.
|
|
161
|
+
When `k-parse` is used without an explicit input pattern or type, the
|
|
162
|
+
wire pattern is derived from the parsed value tree.
|
|
158
163
|
|
|
159
164
|
- An empty textual node is a closed product: `{}`.
|
|
160
165
|
- A textual node with multiple children is a closed product.
|
|
161
|
-
- A textual node with one child is interpreted as an open union by
|
|
162
|
-
|
|
163
|
-
|
|
166
|
+
- A textual node with one child is interpreted as an open union by
|
|
167
|
+
default.
|
|
168
|
+
- An explicit product input pattern may force such a node to be
|
|
169
|
+
treated as a singleton product instead.
|
|
164
170
|
|
|
165
171
|
For example, parsing:
|
|
166
172
|
|
|
@@ -179,18 +185,20 @@ without an input pattern derives:
|
|
|
179
185
|
]
|
|
180
186
|
```
|
|
181
187
|
|
|
182
|
-
During this construction, finite closed pattern subtrees are hash-consed
|
|
183
|
-
the leaves upward. The starting closed leaf is `["closed-product",
|
|
184
|
-
closed nodes collapse only when they have the same closed
|
|
185
|
-
and the same already-collapsed child targets.
|
|
186
|
-
and recursive closed nodes are not
|
|
188
|
+
During this construction, finite closed pattern subtrees are hash-consed
|
|
189
|
+
from the leaves upward. The starting closed leaf is `["closed-product",
|
|
190
|
+
[]]`. Two closed nodes collapse only when they have the same closed
|
|
191
|
+
kind, the same labels, and the same already-collapsed child targets.
|
|
192
|
+
Open nodes keep their identity, and recursive closed nodes are not
|
|
193
|
+
collapsed by this witness-tree rule.
|
|
187
194
|
|
|
188
|
-
This is canonicalization of the pattern graph carried in the wire
|
|
189
|
-
not DAG compression of the value payload.
|
|
195
|
+
This is canonicalization of the pattern graph carried in the wire
|
|
196
|
+
stream. It is not DAG compression of the value payload.
|
|
190
197
|
|
|
191
198
|
## Prefix-Free Value Encoding
|
|
192
199
|
|
|
193
|
-
The value payload is a bitstream interpreted relative to the pattern
|
|
200
|
+
The value payload is a bitstream interpreted relative to the pattern
|
|
201
|
+
graph.
|
|
194
202
|
|
|
195
203
|
Base rules:
|
|
196
204
|
|
|
@@ -209,25 +217,27 @@ width = ceil(log2(cardinality))
|
|
|
209
217
|
emit tag ordinal in width bits
|
|
210
218
|
```
|
|
211
219
|
|
|
212
|
-
This is the simplest canonical prefix-free choice encoding. It is not
|
|
213
|
-
word on compression, but it gives a regular and compact base
|
|
214
|
-
later be refined without changing the abstract
|
|
220
|
+
This is the simplest canonical prefix-free choice encoding. It is not
|
|
221
|
+
the final word on compression, but it gives a regular and compact base
|
|
222
|
+
encoding that can later be refined without changing the abstract
|
|
223
|
+
pattern/value model.
|
|
215
224
|
|
|
216
225
|
## Projections
|
|
217
226
|
|
|
218
227
|
The base tree encoding is chosen so that:
|
|
219
228
|
|
|
220
229
|
- `/tag` only needs to read the union choice at the current node,
|
|
221
|
-
- `.field` follows canonical product order and may later be accelerated
|
|
222
|
-
higher-level indexes if necessary.
|
|
230
|
+
- `.field` follows canonical product order and may later be accelerated
|
|
231
|
+
by higher-level indexes if necessary.
|
|
223
232
|
|
|
224
|
-
In the materialized runtime, projections also project the carried
|
|
225
|
-
`.field` returns the subpattern at that product field, and
|
|
226
|
-
subpattern at that union tag. If no carried subpattern
|
|
227
|
-
encoding can still derive a witness pattern from
|
|
233
|
+
In the materialized runtime, projections also project the carried
|
|
234
|
+
pattern: `.field` returns the subpattern at that product field, and
|
|
235
|
+
`/tag` returns the subpattern at that union tag. If no carried subpattern
|
|
236
|
+
is available, later encoding can still derive a witness pattern from
|
|
237
|
+
the result tree.
|
|
228
238
|
|
|
229
|
-
The primitive codec stays minimal. Projection indexes, framing, or
|
|
230
|
-
schemes are explicitly separate concerns.
|
|
239
|
+
The primitive codec stays minimal. Projection indexes, framing, or
|
|
240
|
+
sharing schemes are explicitly separate concerns.
|
|
231
241
|
|
|
232
242
|
## Design Boundary
|
|
233
243
|
|
|
@@ -240,17 +250,18 @@ The new codec work is split into layers:
|
|
|
240
250
|
|
|
241
251
|
## Files
|
|
242
252
|
|
|
253
|
+
- [`../DOCS/FILE_FORMATS.md`](../DOCS/FILE_FORMATS.md):
|
|
254
|
+
comprehensive guide to file formats and compilation pipelines.
|
|
243
255
|
- [`../DOCS/CODEC_ARCHITECTURE.md`](../DOCS/CODEC_ARCHITECTURE.md):
|
|
244
256
|
comprehensive architecture specification for k codecs.
|
|
245
257
|
- [`../DOCS/CODECS.md`](../DOCS/CODECS.md):
|
|
246
258
|
practical guide to writing, running, and testing codecs.
|
|
247
259
|
- [`POLYMORPHIC_BINARY_FORMAT.md`](./POLYMORPHIC_BINARY_FORMAT.md):
|
|
248
|
-
main design document for the
|
|
260
|
+
main design document for the pattern-plus-prefix-tree codec.
|
|
249
261
|
- [`BINARY_FORMAT.md`](./BINARY_FORMAT.md):
|
|
250
262
|
notes on future binary packaging of the same abstract semantics.
|
|
251
263
|
|
|
252
264
|
## Status
|
|
253
265
|
|
|
254
|
-
The active command-line pipeline emits and consumes the self-hosted
|
|
255
|
-
pattern+value stream.
|
|
256
|
-
|
|
266
|
+
The active command-line pipeline emits and consumes the self-hosted
|
|
267
|
+
binary pattern+value stream.
|
|
@@ -49,24 +49,76 @@ function propertyListToFilter(propertyList, varPrefix = "X") {
|
|
|
49
49
|
return fmt(0);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
function valueToK(
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
52
|
+
function valueToK(root) {
|
|
53
|
+
let result = "";
|
|
54
|
+
const stack = [{
|
|
55
|
+
val: root,
|
|
56
|
+
assign(s) { result = s; }
|
|
57
|
+
}];
|
|
58
|
+
|
|
59
|
+
while (stack.length > 0) {
|
|
60
|
+
const frame = stack.pop();
|
|
61
|
+
if (frame.finishProduct) {
|
|
62
|
+
frame.assign(`{${frame.parts.join(", ")}}`);
|
|
63
|
+
continue;
|
|
59
64
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
65
|
+
if (frame.finishVariant) {
|
|
66
|
+
const base = frame.base;
|
|
67
|
+
const tags = frame.tags;
|
|
68
|
+
frame.assign(tags.length === 0 ? base : `${base}|${tags.reverse().join("|")}`);
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const v = frame.val;
|
|
73
|
+
if (isVariant(v)) {
|
|
74
|
+
const tags = [];
|
|
75
|
+
let curr = v;
|
|
76
|
+
while (isVariant(curr)) {
|
|
77
|
+
tags.push(pLabel(curr.tag).trimStart());
|
|
78
|
+
curr = curr.value;
|
|
79
|
+
}
|
|
80
|
+
const finishFrame = {
|
|
81
|
+
finishVariant: true,
|
|
82
|
+
tags,
|
|
83
|
+
base: "",
|
|
84
|
+
assign: frame.assign
|
|
85
|
+
};
|
|
86
|
+
stack.push(finishFrame);
|
|
87
|
+
stack.push({
|
|
88
|
+
val: curr,
|
|
89
|
+
assign(s) { finishFrame.base = s; }
|
|
90
|
+
});
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (isProduct(v)) {
|
|
95
|
+
const keys = Object.keys(v.product);
|
|
96
|
+
if (keys.length === 0) {
|
|
97
|
+
frame.assign("{}");
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const parts = new Array(keys.length);
|
|
101
|
+
const finishFrame = {
|
|
102
|
+
finishProduct: true,
|
|
103
|
+
parts,
|
|
104
|
+
assign: frame.assign
|
|
105
|
+
};
|
|
106
|
+
stack.push(finishFrame);
|
|
107
|
+
for (let i = keys.length - 1; i >= 0; i--) {
|
|
108
|
+
const k = keys[i];
|
|
109
|
+
const label = pLabel(k);
|
|
110
|
+
stack.push({
|
|
111
|
+
val: v.product[k],
|
|
112
|
+
assign(s) { parts[i] = `${s}${label}`; }
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
frame.assign(String(v));
|
|
68
119
|
}
|
|
69
|
-
|
|
120
|
+
|
|
121
|
+
return result;
|
|
70
122
|
}
|
|
71
123
|
|
|
72
124
|
function valueWithEnvelopeToK(value) {
|