functionalscript 0.34.0 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -28,22 +28,39 @@ Install FunctionalScript via npm:
28
28
  npm install -g functionalscript
29
29
  ```
30
30
 
31
- The FunctionalScript compiler command (`fjs compile`) currently supports:
31
+ The `fjs` CLI provides several commands:
32
32
 
33
- * `import` statements
34
- * `const` declarations
33
+ | Command | Description |
34
+ |------------------|-----------------------------------------------------------|
35
+ | `fjs test` / `t` | Run the FunctionalScript test suite |
36
+ | `fjs compile` / `c` | Compile a `.f.ts` module to JavaScript |
37
+ | `fjs run` / `r` | Run a FunctionalScript module as a Node program |
38
+ | `fjs cas` / `s` | Content-addressable storage (`add`, `get`, `list`) |
39
+ | `fjs mcp` / `m` | Run an MCP server over stdio exposing the CAS as tools |
40
+ | `fjs ci` / `i` | Generate the GitHub Actions CI workflow |
35
41
 
36
- It does **not** yet support functions or complex expressions.
42
+ ### Content-Addressable Storage (CAS)
37
43
 
38
- Example usage with `fjs`:
44
+ FunctionalScript ships a built-in CAS for storing and retrieving blobs by their cryptographic hash:
39
45
 
40
46
  ```bash
41
- fjs compile example.f.js output.json
42
- # or
43
- fjs compile example.f.js output.f.js
47
+ fjs cas add myfile.txt # store a file, print its cBase32 hash
48
+ fjs cas get <hash> # restore a blob by hash
49
+ fjs cas list # list all stored hashes
44
50
  ```
45
51
 
46
- FunctionalScript code can be compiled directly into either JSON or JavaScript without imports.
52
+ Blobs are stored under `~/.cas/` and addressed by their SHA-256 hash encoded in cBase32.
53
+
54
+ ### MCP Server
55
+
56
+ The CAS is also exposed as an [MCP](https://modelcontextprotocol.io/) server so LLM agents can read and write blobs without a shell:
57
+
58
+ ```bash
59
+ # register with Claude CLI
60
+ claude mcp add cas -- npx functionalscript m
61
+ ```
62
+
63
+ See [`fs/cas/mcp/README.md`](fs/cas/mcp/README.md) for details on the `cas_add`, `cas_get`, and `cas_list` tools.
47
64
 
48
65
  ## Vision
49
66
 
@@ -67,10 +84,6 @@ In FunctionalScript:
67
84
  - A module can depend only on another FunctionalScript module.
68
85
  - It also has no standard library. Only a safe subset of standard JavaScript API can be used without referencing other modules.
69
86
 
70
- ## Our Next Step
71
-
72
- [Re-architecture of NaNVM](https://medium.com/@sergeyshandar/nanvm-re-architecture-8097f766ec1c?sk=d14ec1daf73ac5442f12ce20b2bc037a).
73
-
74
87
  ## Sponsors
75
88
 
76
89
  - [KirillOsenkov](https://github.com/KirillOsenkov),
@@ -8,4 +8,5 @@ export declare const proof: {
8
8
  invalidInput: () => void;
9
9
  validPadding: () => void;
10
10
  knownVectors: () => void;
11
+ decodeOverflow: () => void;
11
12
  };
@@ -1,14 +1,9 @@
1
+ import { assertEq } from "../asserts/module.f.js";
1
2
  import { empty, vec } from "../types/bit_vec/module.f.js";
2
3
  import { encode, decode } from "./module.f.js";
3
4
  const check = (s, v) => {
4
- const sr = encode(v);
5
- if (sr !== s) {
6
- throw ['encode', sr, s];
7
- }
8
- const vr = decode(s);
9
- if (vr !== v) {
10
- throw ['decode', vr, v];
11
- }
5
+ assertEq(encode(v), s);
6
+ assertEq(decode(s), v);
12
7
  };
13
8
  export const proof = {
14
9
  empty: () => {
@@ -46,62 +41,29 @@ export const proof = {
46
41
  },
47
42
  nonOctet: () => {
48
43
  // encode rejects bit vectors whose length is not a multiple of 8
49
- if (encode(vec(1n)(0n)) !== null) {
50
- throw '1-bit input should return null';
51
- }
52
- if (encode(vec(6n)(0n)) !== null) {
53
- throw '6-bit input should return null';
54
- }
55
- if (encode(vec(12n)(0n)) !== null) {
56
- throw '12-bit input should return null';
57
- }
44
+ assertEq(encode(vec(1n)(0n)), null);
45
+ assertEq(encode(vec(6n)(0n)), null);
46
+ assertEq(encode(vec(12n)(0n)), null);
58
47
  },
59
48
  invalidInput: () => {
60
- if (decode('!') !== null) {
61
- throw 'invalid char should return null';
62
- }
63
- if (decode('A!AA') !== null) {
64
- throw 'invalid char mid-string should return null';
65
- }
66
- if (decode('A===') !== null) {
67
- throw 'three equals signs should return null';
68
- }
69
- if (decode('A') !== null) {
70
- throw 'length 1 (not multiple of 4) should return null';
71
- }
72
- if (decode('AA') !== null) {
73
- throw 'length 2 (no padding) should return null';
74
- }
75
- if (decode('AAA') !== null) {
76
- throw 'length 3 (no padding) should return null';
77
- }
78
- if (decode('=AAA') !== null) {
79
- throw 'padding not at end should return null';
80
- }
49
+ assertEq(decode('!'), null);
50
+ assertEq(decode('A!AA'), null);
51
+ assertEq(decode('A==='), null);
52
+ assertEq(decode('A'), null);
53
+ assertEq(decode('AA'), null);
54
+ assertEq(decode('AAA'), null);
55
+ assertEq(decode('=AAA'), null);
81
56
  // Non-zero padding bits must be rejected (RFC 4648 §3.5)
82
- if (decode('AB==') !== null) {
83
- throw 'non-zero 4-pad-bit should return null';
84
- }
85
- if (decode('AAB=') !== null) {
86
- throw 'non-zero 2-pad-bit should return null';
87
- }
57
+ assertEq(decode('AB=='), null);
58
+ assertEq(decode('AAB='), null);
88
59
  },
89
60
  validPadding: () => {
90
61
  // "==" padding
91
- const v2 = decode('AA==');
92
- if (v2 !== vec(8n)(0n)) {
93
- throw ['AA== should decode to 8-bit zero', v2];
94
- }
62
+ assertEq(decode('AA=='), vec(8n)(0n));
95
63
  // "=" padding
96
- const v1 = decode('AAA=');
97
- if (v1 !== vec(16n)(0n)) {
98
- throw ['AAA= should decode to 16-bit zero', v1];
99
- }
64
+ assertEq(decode('AAA='), vec(16n)(0n));
100
65
  // No padding
101
- const v0 = decode('AAAA');
102
- if (v0 !== vec(24n)(0n)) {
103
- throw ['AAAA should decode to 24-bit zero', v0];
104
- }
66
+ assertEq(decode('AAAA'), vec(24n)(0n));
105
67
  },
106
68
  knownVectors: () => {
107
69
  // RFC 4648 §10 test vectors (byte-aligned, treating bytes as 8-bit MSB vectors)
@@ -114,4 +76,11 @@ export const proof = {
114
76
  // 0x66 0x6f 0x6f ('foo') → Zm9v
115
77
  check('Zm9v', vec(24n)(0x666f6fn));
116
78
  },
79
+ decodeOverflow: () => {
80
+ // 174_764 base64 chars decode to 1_048_584 bits, 8 over `maxLength`;
81
+ // `decode` should return `null` instead of throwing on (or hanging
82
+ // while building) an oversized `bigint`.
83
+ const oversized = 'A'.repeat(174_764);
84
+ assertEq(decode(oversized), null);
85
+ },
117
86
  };
@@ -11,8 +11,10 @@
11
11
  *
12
12
  * @module
13
13
  */
14
- import { msb, length, vec, empty } from "../types/bit_vec/module.f.js";
15
- const { popFront, concat } = msb;
14
+ import { msb, lsb, length, vec } from "../types/bit_vec/module.f.js";
15
+ import {} from "../types/list/module.f.js";
16
+ const { popFront } = msb;
17
+ const { tryListToVec: reversedListToVec } = lsb;
16
18
  /**
17
19
  * Builds a {@link BaseN} codec for a fixed chunk width and alphabet.
18
20
  *
@@ -40,15 +42,19 @@ export const baseN = (bits, alphabet, normalize) => {
40
42
  return result;
41
43
  },
42
44
  stringToVec: s => {
43
- let result = empty;
45
+ // Build a reversed chunk list, bailing out at the first invalid
46
+ // character so malformed input is rejected in O(prefix) time and
47
+ // `normalize` is never run past it. `listToVec` then concatenates in
48
+ // O(n log n).
49
+ let chunks = null;
44
50
  for (const c of s) {
45
51
  const index = toIndex(c);
46
52
  if (index < 0) {
47
53
  return null;
48
54
  }
49
- result = concat(result)(vecN(BigInt(index)));
55
+ chunks = { first: vecN(BigInt(index)), tail: chunks };
50
56
  }
51
- return result;
57
+ return reversedListToVec(chunks);
52
58
  },
53
59
  };
54
60
  };
@@ -7,4 +7,5 @@ export declare const proof: {
7
7
  decodeInvalid: () => void;
8
8
  normalizeHit: () => void;
9
9
  normalizeMiss: () => void;
10
+ big: () => void;
10
11
  };
@@ -1,4 +1,5 @@
1
- import { empty, vec } from "../types/bit_vec/module.f.js";
1
+ import { assertEq } from "../asserts/module.f.js";
2
+ import { empty, maxLength, vec, length } from "../types/bit_vec/module.f.js";
2
3
  import { baseN } from "./module.f.js";
3
4
  const hex = baseN(4n, '0123456789abcdef');
4
5
  const cb32 = baseN(5n, '0123456789abcdefghjkmnpqrstvwxyz', c => {
@@ -18,6 +19,9 @@ const cb32 = baseN(5n, '0123456789abcdefghjkmnpqrstvwxyz', c => {
18
19
  }
19
20
  }
20
21
  });
22
+ // Sample input for the `big` proof below: 262 144 `f` characters decode into a
23
+ // 1 Mibit (`maxLength`) vector.
24
+ const bigSampleHex = `f`.repeat(Number(maxLength >> 2n));
21
25
  export const proof = {
22
26
  encodeEmpty: () => {
23
27
  const s = hex.vecToString(empty);
@@ -83,4 +87,11 @@ export const proof = {
83
87
  throw 'unknown char should return null';
84
88
  }
85
89
  },
90
+ // Decodes a 1 Mibit hex string. With the O(n log n) `listToVec` builder this
91
+ // runs in well under a second (was ~13 s node / ~43 s bun under the old
92
+ // per-chunk `concat`).
93
+ big: () => {
94
+ const x = hex.stringToVec(bigSampleHex);
95
+ assertEq(length(x), maxLength);
96
+ }
86
97
  };
@@ -47,11 +47,14 @@
47
47
  * only metadata was asked for; streaming detection returns correct
48
48
  * `{ length, mime_type, type[, url] }` regardless of size.
49
49
  *
50
- * **`content: true`** materializes the bytes (via `collectRead`, bounded by
51
- * `maxLength`), classifies them with the same machine via `fs/mime` `detectVec`,
52
- * then encodes the inline payload by `type` a `fs/text/utf8` `fromVec` string for
53
- * `text`, base64 for `base64`. A blob larger than `maxLength` is unsupported here
54
- * and should be fetched via `url`.
50
+ * **`content: true`** first derives `{ length, mime_type, type }` with the same
51
+ * size-independent `fs/mime` `detectStream` machine, then materializes the bytes
52
+ * (via `collectRead`, bounded by `maxLength`) and encodes the inline payload by
53
+ * `type` — a `fs/text/utf8` `fromVec` string for `text`, base64 for `base64`. A
54
+ * blob larger than `maxLength` (128 KiB) cannot be buffered into one `Vec`, so it
55
+ * is rejected here with a descriptive *"too large"* error (carrying the byte size
56
+ * and `url`) rather than being misreported as absent; it should be fetched via
57
+ * `url` or inspected with metadata-only `cas_get`.
55
58
  *
56
59
  * ## Encoding split: hashes vs. content
57
60
  *
@@ -69,6 +72,8 @@
69
72
  * - `type: 'base64'` with malformed content (base64 `decode` → `null`) → `isError`
70
73
  * - malformed `hash` (`cBase32ToVec` → `null`) → `isError`
71
74
  * - `cas_get` on an absent hash (`c.read` → `undefined`) → `isError`
75
+ * - `cas_get` with `content: true` on a blob larger than `maxLength` → `isError`
76
+ * (distinct "too large" message, not "no such hash")
72
77
  * - unknown tool `name` → `isError`
73
78
  *
74
79
  * @module
@@ -79,9 +84,9 @@ import { pure } from "../../effects/module.f.js";
79
84
  import { create } from "../../effects/memory/module.f.js";
80
85
  import { cBase32ToVec, vecToCBase32 } from "../../cbase32/module.f.js";
81
86
  import { decode as base64Decode, encode as base64Encode } from "../../base64/module.f.js";
82
- import { utf8 } from "../../text/module.f.js";
83
- import { detectVec, detectStream } from "../../mime/module.f.js";
84
- import { empty, length as bitVecLength, maxLength, msb } from "../../types/bit_vec/module.f.js";
87
+ import { tryUtf8 } from "../../text/module.f.js";
88
+ import { detectStream } from "../../mime/module.f.js";
89
+ import { empty, length as bitVecLength, maxLength, maxLengthBytes, msb } from "../../types/bit_vec/module.f.js";
85
90
  import { ok, error } from "../../types/result/module.f.js";
86
91
  import { rm } from "../../effects/node/module.f.js";
87
92
  import { stdioTransport } from "../../mcp/stdio/module.f.js";
@@ -139,7 +144,7 @@ const casToolRegistry = (home) => {
139
144
  const c = fileCas(sha256)(home);
140
145
  const casUploadDir = `${home}/cas_upload`;
141
146
  return [
142
- toolEntry('cas_add', 'Store content and return its hash (cBase32). Pass type:"base64" for binary; type:"url" to stream a file from $HOME/cas_upload/ (no size limit); omit or pass type:"text" for UTF-8 text (default).', casAddArgs, ({ type, content }) => {
147
+ toolEntry('cas_add', 'Store content and return its hash (cBase32). Pass type:"base64" for binary; omit or pass type:"text" for UTF-8 text (default). Inline content (text/base64) is capped at 128 KiB (131072 bytes) — larger content is rejected. For larger content use type:"url" to stream a file from $HOME/cas_upload/ (no size limit).', casAddArgs, ({ type, content }) => {
143
148
  // type:'url' — stream the file into cas, then delete the source on success
144
149
  if (type === 'url') {
145
150
  if (!content.startsWith(`${casUploadDir}/`) || content.includes('..')) {
@@ -150,66 +155,56 @@ const casToolRegistry = (home) => {
150
155
  : rm(content).step(() => pure(okResult(vecToCBase32(v)))));
151
156
  }
152
157
  // type:'text' or 'base64' — resolve content to Vec, store via c.write()
153
- let x;
154
- switch (type) {
155
- case 'base64':
156
- const value = base64Decode(content);
157
- x = pure(value === null ? `invalid base64 content: ${content}` : value);
158
- break;
159
- default:
160
- x = pure(utf8(content));
161
- break;
162
- }
163
- return x.step(value => typeof value === 'string'
164
- ? pure(errorResult(value))
158
+ let x = type === 'base64'
159
+ ? base64Decode(content)
160
+ : tryUtf8(content);
161
+ return x === null
162
+ ? pure(errorResult('too large or malformed use type:"url" for large content'))
165
163
  // The resolved content fits in one chunk; feed it as a single-item stream.
166
- : c.write(nonEmpty(ok(value), elEmpty())).step(hashResult => pure(hashResult[0] === 'error'
164
+ : c.write(nonEmpty(ok(x), elEmpty())).step(([tag, hash]) => pure(tag === 'error'
167
165
  ? errorResult('write')
168
- : okResult(vecToCBase32(hashResult[1])))));
166
+ : okResult(vecToCBase32(hash))));
169
167
  }),
170
- toolEntry('cas_get', 'Inspect a blob by hash. Always returns JSON {length,mime_type,type[,url]} where type is "text" or "base64". Pass content:true to also include the inline content string.', casGetArgs, r => {
168
+ toolEntry('cas_get', 'Inspect a blob by hash. Always returns JSON {length,mime_type,type[,url]} where type is "text" or "base64". Pass content:true to also include the inline content string, but content is capped at 128 KiB (131072 bytes) — a larger blob is rejected with an error. To download a blob, prefer the url field returned in the result instead of requesting inline content.', casGetArgs, r => {
171
169
  const key = cBase32ToVec(r.hash);
172
170
  if (key === null) {
173
171
  return pure(errorResult(`invalid cBase32 hash: ${r.hash}`));
174
172
  }
175
173
  const url = c.url(key);
176
- // Metadata-only (the default): derive `{ length, mime_type, type }` by
177
- // streaming the blob through the detector state machine. Never buffers
178
- // the blob, so this is size-independent — large blobs that overflow a
179
- // single `Vec` still return correct metadata instead of an error.
180
- if (r.content !== true) {
181
- return detectStream(c.read(key)).step(result => {
182
- if (result[0] === 'error') {
183
- return pure(errorResult(`no such hash: ${r.hash}`));
184
- }
185
- const { length, mime_type, type } = result[1];
186
- const meta = { length: Number(length), mime_type, type, url };
187
- return pure(okResult(toJson(meta)));
188
- });
189
- }
190
- // content:true — collect the bytes (bounded by `maxLength`) so they can
191
- // be encoded inline. Classification uses the *same* `detectVec` state
192
- // machine as the metadata-only path; `type` then selects the encoding —
193
- // a UTF-8 string for `text`, base64 for `base64`.
194
- return collectRead(c.read(key)).step(result => {
195
- if (result[0] === 'error') {
174
+ return detectStream(c.read(key)).step(([tag, detected]) => {
175
+ if (tag === 'error') {
196
176
  return pure(errorResult(`no such hash: ${r.hash}`));
197
177
  }
198
- const value = result[1];
199
- const { length, mime_type, type } = detectVec(value);
178
+ const { length, mime_type, type } = detected;
200
179
  const meta = { length: Number(length), mime_type, type, url };
201
- if (type === 'text') {
202
- // `type: 'text'` means the detector validated `value` as UTF-8, so
203
- // `fromVec` is non-null here; guard defensively regardless.
204
- const str = fromVec(value);
205
- return pure(str === null
206
- ? errorResult(`content is not byte-aligned: ${r.hash}`)
207
- : okResult(toJson({ ...meta, content: str })));
180
+ if (r.content !== true) {
181
+ // content:true path continues below; this is just the metadata step.
182
+ return pure(okResult(toJson(meta)));
183
+ }
184
+ // A single `Vec` caps at `maxLength` bits (`maxLengthBytes` bytes), so
185
+ // a larger blob cannot be buffered for inline transfer. Report the
186
+ // size and point at the size-independent alternatives instead of
187
+ // misreporting an existing blob as `no such hash`.
188
+ if (length > maxLengthBytes) {
189
+ return pure(errorResult(`blob too large to fetch inline (${length} bytes, limit ${maxLengthBytes} bytes); use the url field (${url}) or omit content for metadata`));
208
190
  }
209
- const blob = base64Encode(value);
210
- return pure(blob === null
211
- ? errorResult(`content is not byte-aligned: ${r.hash}`)
212
- : okResult(toJson({ ...meta, content: blob })));
191
+ return collectRead(c.read(key)).step(([collectTag, value]) => {
192
+ if (collectTag === 'error') {
193
+ return pure(errorResult(`no such hash: ${r.hash}`));
194
+ }
195
+ if (type === 'text') {
196
+ // `type: 'text'` means the detector validated `value` as UTF-8,
197
+ // so `fromVec` is non-null here; guard defensively regardless.
198
+ const str = fromVec(value);
199
+ return pure(str === null
200
+ ? errorResult(`content is not byte-aligned: ${r.hash}`)
201
+ : okResult(toJson({ ...meta, content: str })));
202
+ }
203
+ const blob = base64Encode(value);
204
+ return pure(blob === null
205
+ ? errorResult(`content is not byte-aligned: ${r.hash}`)
206
+ : okResult(toJson({ ...meta, content: blob })));
207
+ });
213
208
  });
214
209
  }),
215
210
  toolEntry('cas_list', 'List all stored content hashes (cBase32), one per line.', casListArgs, () => c.list().step(hashes => pure(okResult(hashes.map(vecToCBase32).join('\n'))))),
@@ -2,6 +2,9 @@ export declare const proof: {
2
2
  getMetaLargeMultiChunkBlobNoError: () => void;
3
3
  getMetaLargeMultiChunkUtf8Symbols: () => void;
4
4
  getMetaLargeMultiChunkTextThenBinary: () => void;
5
+ getContentLargeBlobTooLargeError: () => void;
6
+ getContentMissingHashIsError: () => void;
7
+ getContentDoubleEscapedOverflowWritesInternalError: () => void;
5
8
  toolsListAdvertisesThreeTools: () => void;
6
9
  addReturnsHash: () => void;
7
10
  textAddGetMetaRoundTrips: () => void;
@@ -14,6 +17,10 @@ export declare const proof: {
14
17
  listEnumeratesStoredHashes: () => void;
15
18
  addInvalidContentIsError: () => void;
16
19
  addBadLengthContentIsError: () => void;
20
+ addBase64OverLimitIsError: () => void;
21
+ addTextOverLimitIsError: () => void;
22
+ addTextAtLimitSucceeds: () => void;
23
+ addBase64AtLimitIsError: () => void;
17
24
  getUnterminatedHashIsError: () => void;
18
25
  addMissingContentIsError: () => void;
19
26
  getMissingHashArgumentIsError: () => void;
@@ -11,6 +11,8 @@ import { mcpStep, uninitializedState, } from "../../mcp/module.f.js";
11
11
  import { emptyState, virtual } from "../../effects/node/virtual/module.f.js";
12
12
  import { casConfig, casMcpHandlers } from "./module.f.js";
13
13
  import { ok as resultOk } from "../../types/result/module.f.js";
14
+ import { stdioTransport } from "../../mcp/stdio/module.f.js";
15
+ import { fromVec } from "../../types/uint8array/module.f.js";
14
16
  const initialTestState = { memory: { next: 0, values: {} } };
15
17
  const mock = {
16
18
  memCreate: value => state => {
@@ -65,6 +67,23 @@ const call = (id, name, args) => ({ jsonrpc: '2.0', method: 'tools/call', id, pa
65
67
  const list = (id) => ({ jsonrpc: '2.0', method: 'tools/list', id });
66
68
  // Runs `init`, `notifications/initialized`, then `msgs`; returns the tool responses.
67
69
  const session = (...msgs) => runSessionVirtual({}, '/home/user')([init, initialized, ...msgs]).slice(2);
70
+ // UTF-8 bytes of `s` as a plain array — the virtual stdin byte stream.
71
+ const toBytes = (s) => [...fromVec(utf8(s))];
72
+ // Runs `init`, `notifications/initialized`, then `msgs` through the *real*
73
+ // stdio pipeline — `mcpStep` wrapped in `stdioTransport`, driving
74
+ // `writeResponse`'s `tryUtf8` encode over actual stdin/stdout bytes. Unlike
75
+ // `session`/`runSessionVirtual`, which collect `mcpStep`'s `Response` values
76
+ // as plain JS objects and never serialize/encode them, this is the only
77
+ // helper that can observe the transport's oversized-response fallback (see
78
+ // `fs/mcp/stdio/module.f.ts` `writeResponse`).
79
+ const runStdio = (root, home = '/home/user') => (msgs) => {
80
+ const input = [init, initialized, ...msgs].map(m => JSON.stringify(m)).join('\n') + '\n';
81
+ const effect = create(uninitializedState).step(sessionKey => stdioTransport(mcpStep(casConfig)(casMcpHandlers(home))(sessionKey)));
82
+ const stdout = virtual({ ...emptyState, root, stdin: toBytes(input) })(effect)[0].stdout;
83
+ // Only requests get a written line (notifications, like `initialized`,
84
+ // write nothing) — drop the `init` response, keep one line per `msgs` entry.
85
+ return stdout.split('\n').filter(line => line.length > 0).slice(1).map(line => JSON.parse(line));
86
+ };
68
87
  const resultOf = (resp) => resp.result;
69
88
  const item0 = (resp) => resultOf(resp).content[0];
70
89
  const textOf = (resp) => item0(resp).text;
@@ -75,6 +94,22 @@ const binarySample = base64Encode(vec8(0x2an));
75
94
  // A base64 blob whose leading bytes are the PNG magic-byte signature, so
76
95
  // `cas_get` detects its type and returns base64 with mime_type image/png.
77
96
  const pngSample = base64Encode(u8ListToVec(msb)([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x01]));
97
+ // Returns the RFC 4648 base64 encoding of `n` zero bytes, computed directly
98
+ // without bigint arithmetic — safe near maxLengthBytes where base64Encode on a
99
+ // padded Vec would trigger the encode-padding-overflow bug. Handles every n%3
100
+ // residue: rem=0 → no padding, rem=1 → 'AA==', rem=2 → 'AAA='.
101
+ const base64OfA = (n) => {
102
+ const groups = Number(n / 3n);
103
+ const rem = Number(n % 3n);
104
+ const body = 'AAAA'.repeat(groups);
105
+ if (rem === 0) {
106
+ return body;
107
+ }
108
+ if (rem === 1) {
109
+ return body + 'AA==';
110
+ }
111
+ return body + 'AAA=';
112
+ };
78
113
  // ── Tests ───────────────────────────────────────────────────────────────────────
79
114
  // A blob larger than one read chunk (> 128 KiB) used to fail metadata-only
80
115
  // cas_get because `collectRead` overflowed `maxLength`. With `detectStream`,
@@ -107,6 +142,13 @@ const asciiChunk = repeat(maxLengthBytes)(vec8(0x61n));
107
142
  const symbolChunk = repeat(maxLengthBytes / 2n)(u8ListToVec(msb)([0xc3, 0xa9]));
108
143
  // A full chunk of 0xFF — an invalid UTF-8 lead byte, so binary (no magic match).
109
144
  const binaryChunk = repeat(maxLengthBytes)(vec8(0xffn));
145
+ // 33,000 `"` bytes (0x22): valid UTF-8, so `cas_get` classifies it as text and
146
+ // takes the double-JSON-escaping path (`toJson` builds the inner CAS JSON, then
147
+ // the outer `stringifyJson` embeds that as the `text` field). Each quote costs
148
+ // 4 final bytes (2 on each escaping pass), so 33,000 of them alone push the
149
+ // encoded line past `maxLength` (131,072 bytes) — well above the strict 32,769
150
+ // minimum needed, without approaching any other size-related edge case.
151
+ const quotesChunk = repeat(33000n)(vec8(0x22n));
110
152
  export const proof = {
111
153
  // Both chunks repeated ASCII → text/plain, no error on a > 128 KiB blob.
112
154
  getMetaLargeMultiChunkBlobNoError: largeMultiChunkBlobMeta(asciiChunk, asciiChunk, 'text', 'text/plain'),
@@ -115,6 +157,58 @@ export const proof = {
115
157
  // First chunk valid UTF-8, second chunk binary → base64/octet-stream. The
116
158
  // streaming validator must see the trailing binary chunk to reject text.
117
159
  getMetaLargeMultiChunkTextThenBinary: largeMultiChunkBlobMeta(asciiChunk, binaryChunk, 'base64', 'application/octet-stream'),
160
+ // content:true on a present blob larger than `maxLength` reports a distinct
161
+ // "too large" error, not "no such hash" — the blob exists, it just can't be
162
+ // buffered inline. The metadata-only path (above) still returns its size/type.
163
+ getContentLargeBlobTooLargeError: () => {
164
+ const root = { 'home': { 'user': { 'cas_upload': { 'big': [asciiChunk, asciiChunk] } } } };
165
+ const [addResp] = runSessionVirtual(root)([
166
+ init, initialized,
167
+ call(2, 'cas_add', { content: '/home/user/cas_upload/big', type: 'url' }),
168
+ ]).slice(2);
169
+ const hash = textOf(addResp);
170
+ const [, getResp] = runSessionVirtual(root)([
171
+ init, initialized,
172
+ call(2, 'cas_add', { content: '/home/user/cas_upload/big', type: 'url' }),
173
+ call(3, 'cas_get', { hash, content: true }),
174
+ ]).slice(2);
175
+ assertEq(resultOf(getResp).isError, true);
176
+ const text = textOf(getResp);
177
+ assert(text.includes('too large'));
178
+ assert(!text.includes('no such hash'));
179
+ },
180
+ // content:true on a genuinely absent hash still reports "no such hash" — the
181
+ // oversized branch above must not absorb the absent case.
182
+ getContentMissingHashIsError: () => {
183
+ const absent = vecToCBase32(vec8(0x55n));
184
+ const [resp] = session(call(2, 'cas_get', { hash: absent, content: true }));
185
+ assertEq(resultOf(resp).isError, true);
186
+ assert(textOf(resp).includes('no such hash'));
187
+ },
188
+ // A text blob made entirely of `"` characters keeps the *raw* content
189
+ // (33,000 bytes) well under `maxLengthBytes`, but each quote costs 2 bytes on
190
+ // the inner CAS-JSON escape and 4 bytes once the outer JSON-RPC envelope
191
+ // re-escapes that backslash-quote pair — no `cas_get`-level size guard could
192
+ // see this coming, only the transport's real encode attempt can. Uploaded via
193
+ // `type: 'url'` (a file, not an inlined JSON string) so the test exercises
194
+ // `cas_get`'s response-side escaping without also paying for parsing a huge
195
+ // inline `content` argument on the request side. Confirms `writeResponse`'s
196
+ // `tryUtf8` fallback (not a crash) handles double-escaping overflow, again
197
+ // preserving the request's `id`.
198
+ getContentDoubleEscapedOverflowWritesInternalError: () => {
199
+ const root = { 'home': { 'user': { 'cas_upload': { 'q': [quotesChunk] } } } };
200
+ const [addResp] = runStdio(root)([
201
+ call(2, 'cas_add', { content: '/home/user/cas_upload/q', type: 'url' }),
202
+ ]);
203
+ const hash = textOf(addResp);
204
+ const [, getResp] = runStdio(root)([
205
+ call(2, 'cas_add', { content: '/home/user/cas_upload/q', type: 'url' }),
206
+ call(3, 'cas_get', { hash, content: true }),
207
+ ]);
208
+ const err = getResp;
209
+ assertEq(err.error?.code, -32603);
210
+ assertEq(err.id, 3);
211
+ },
118
212
  toolsListAdvertisesThreeTools: () => {
119
213
  const [resp] = runSessionVirtual({})([init, initialized, list(2)]).slice(2);
120
214
  const tools = resp.result.tools;
@@ -216,6 +310,37 @@ export const proof = {
216
310
  const [resp] = session(call(2, 'cas_add', { content: 'A', type: 'base64' }));
217
311
  assertEq(resultOf(resp).isError, true);
218
312
  },
313
+ // 174_764 'A' chars × 6 bits = 1_048_584 bits, 8 over maxLength — base64Decode
314
+ // returns null and cas_add surfaces an error.
315
+ addBase64OverLimitIsError: () => {
316
+ const [resp] = session(call(2, 'cas_add', { content: 'A'.repeat(174_764), type: 'base64' }));
317
+ assertEq(resultOf(resp).isError, true);
318
+ assert(textOf(resp).includes('too large or malformed'));
319
+ },
320
+ // One ASCII byte past maxLengthBytes — tryUtf8 returns null and cas_add
321
+ // surfaces an error.
322
+ addTextOverLimitIsError: () => {
323
+ const [resp] = session(call(2, 'cas_add', { content: 'a'.repeat(Number(maxLengthBytes) + 1) }));
324
+ assertEq(resultOf(resp).isError, true);
325
+ assert(textOf(resp).includes('too large or malformed'));
326
+ },
327
+ // Exactly maxLengthBytes (131,072) ASCII bytes — tryUtf8 returns a non-null
328
+ // Vec (len === maxLength, not over), so cas_add stores cleanly.
329
+ addTextAtLimitSucceeds: () => {
330
+ const [resp] = session(call(2, 'cas_add', { content: 'a'.repeat(Number(maxLengthBytes)) }));
331
+ assert(!resultOf(resp).isError);
332
+ assert(textOf(resp).length > 0);
333
+ },
334
+ // base64 of exactly maxLengthBytes — currently returns isError because
335
+ // base64Decode measures the raw pre-trim body (1_048_578 bits) against
336
+ // maxLength before stripping the 2 padding bits that would land it exactly at
337
+ // maxLength. Assert the current failing behavior; flip to a success assertion
338
+ // once `fs/base64/todo/decode-rejects-max-size-input.md` is fixed.
339
+ addBase64AtLimitIsError: () => {
340
+ const [resp] = session(call(2, 'cas_add', { content: base64OfA(maxLengthBytes), type: 'base64' }));
341
+ assertEq(resultOf(resp).isError, true);
342
+ assert(textOf(resp).includes('too large or malformed'));
343
+ },
219
344
  getUnterminatedHashIsError: () => {
220
345
  const [resp] = session(call(2, 'cas_get', { hash: '0' }));
221
346
  assertEq(resultOf(resp).isError, true);
@@ -18,6 +18,14 @@
18
18
  * `null`) is written rather than silently discarded, per JSON-RPC 2.0 §5.
19
19
  * - the step yields `null` (a notification needing no reply) → nothing is
20
20
  * written and the loop continues.
21
+ * - a response that doesn't fit in one encoded line (`tryUtf8` overflow —
22
+ * `maxLength`, 128 KiB) never throws. `writeResponse` retries with a fixed,
23
+ * small `-32603` internal-error body carrying the original `id`; if even
24
+ * that overflows (a pathological caller-controlled `id`, e.g. a very large
25
+ * string), it retries once more with `id: null` — a fully constant response
26
+ * shape that is always small enough to encode. Every request that reaches
27
+ * `step` therefore gets *some* response line, never silence and never a
28
+ * crashed process.
21
29
  *
22
30
  * @module
23
31
  */
@@ -18,25 +18,39 @@
18
18
  * `null`) is written rather than silently discarded, per JSON-RPC 2.0 §5.
19
19
  * - the step yields `null` (a notification needing no reply) → nothing is
20
20
  * written and the loop continues.
21
+ * - a response that doesn't fit in one encoded line (`tryUtf8` overflow —
22
+ * `maxLength`, 128 KiB) never throws. `writeResponse` retries with a fixed,
23
+ * small `-32603` internal-error body carrying the original `id`; if even
24
+ * that overflows (a pathological caller-controlled `id`, e.g. a very large
25
+ * string), it retries once more with `id: null` — a fully constant response
26
+ * shape that is always small enough to encode. Every request that reaches
27
+ * `step` therefore gets *some* response line, never silence and never a
28
+ * crashed process.
21
29
  *
22
30
  * @module
23
31
  */
24
32
  import { pure } from "../../effects/module.f.js";
25
33
  import { readLine, write } from "../../effects/node/module.f.js";
26
- import { utf8 } from "../../text/module.f.js";
34
+ import { tryUtf8 } from "../../text/module.f.js";
27
35
  import { stringToList } from "../../text/utf16/module.f.js";
28
36
  import { stringify } from "../../json/module.f.js";
29
37
  import { tokenize } from "../../json/tokenizer/module.f.js";
30
38
  import { parse } from "../../json/parser/module.f.js";
31
39
  import { sort } from "../../types/object/module.f.js";
32
- import { jsonrpc, parseError } from "../../json/rpc/module.f.js";
40
+ import { internalError, jsonrpc, parseError } from "../../json/rpc/module.f.js";
41
+ import { error, ok } from "../../types/result/module.f.js";
33
42
  const stringifyJson = stringify(sort);
34
43
  /** The parse-error response (`-32700`, `id: null`) for a malformed input line. */
35
44
  const parseErrorResponse = { jsonrpc, error: parseError, id: null };
45
+ /** An internal-error response (`-32603`) carrying `id`. */
46
+ const internalErrorResponse = (id) => ({ jsonrpc, error: internalError, id });
36
47
  /** Encodes a response as a newline-terminated UTF-8 line and writes it to `stdout`. */
37
- const writeResponse = (resp) => write('stdout', utf8(stringifyJson(resp) + '\n'));
38
- /** Writes `resp` when present, otherwise does nothing (notification). */
39
- const writeMaybe = (resp) => resp === null ? pure(undefined) : writeResponse(resp);
48
+ const writeResponse = (resp) => {
49
+ const v = tryUtf8(stringifyJson(resp) + '\n');
50
+ return v === null
51
+ ? pure(error(undefined))
52
+ : write('stdout', v).step(() => pure(ok(undefined)));
53
+ };
40
54
  /**
41
55
  * Drives the read-parse-dispatch-write loop for `step` over stdin/stdout.
42
56
  *
@@ -50,5 +64,18 @@ const handleLine = (step) => (line) => {
50
64
  const [t, value] = parse(tokenize(stringToList(line)));
51
65
  return (t === 'error'
52
66
  ? writeResponse(parseErrorResponse)
53
- : step(value).step(writeMaybe)).step(() => stdioTransport(step));
67
+ : step(value).step(resp => resp === null
68
+ ? pure(undefined)
69
+ : writeResponse(resp).step(([t2]) => t2 === 'error'
70
+ // The real response didn't fit. Retry with a fixed, small
71
+ // internal-error body carrying `resp.id` — but a
72
+ // caller-controlled `id` (e.g. a very large string) can
73
+ // itself push even this fallback over `maxLength`, so
74
+ // that retry is bounded by one more: an `id: null`
75
+ // internal-error, whose fully-constant shape is the only
76
+ // line in this transport guaranteed to always encode.
77
+ ? writeResponse(internalErrorResponse(resp.id)).step(([t3]) => t3 === 'error'
78
+ ? writeResponse(internalErrorResponse(null)).step(() => pure(undefined))
79
+ : pure(undefined))
80
+ : pure(undefined)))).step(() => stdioTransport(step));
54
81
  };
@@ -6,5 +6,8 @@ export declare const proof: {
6
6
  malformedJsonWritesParseError: () => void;
7
7
  trailingCommaWritesParseError: () => void;
8
8
  undefinedFieldOmitted: () => void;
9
+ oversizedResponseWritesInternalError: () => void;
10
+ loopContinuesAfterOversizedResponse: () => void;
11
+ oversizedIdFallsBackToNullId: () => void;
9
12
  multipleLines: () => void;
10
13
  };
@@ -4,8 +4,9 @@ import { emptyState, virtual } from "../../effects/node/virtual/module.f.js";
4
4
  import { stringify } from "../../json/module.f.js";
5
5
  import { utf8 } from "../../text/module.f.js";
6
6
  import { fromVec } from "../../types/uint8array/module.f.js";
7
+ import { maxLengthBytes } from "../../types/bit_vec/module.f.js";
7
8
  import { sort } from "../../types/object/module.f.js";
8
- import { jsonrpc, parseError } from "../../json/rpc/module.f.js";
9
+ import { internalError, jsonrpc, parseError } from "../../json/rpc/module.f.js";
9
10
  import { stdioTransport } from "./module.f.js";
10
11
  const stringifyJson = stringify(sort);
11
12
  // Extracts the request `id` (a request has one; a notification does not).
@@ -29,7 +30,11 @@ const runStep = (step) => (input) => virtual({ ...emptyState, stdin: toBytes(inp
29
30
  const run = runStep(echoStep);
30
31
  const okResponse = (id) => stringifyJson({ jsonrpc, result: { ok: true }, id }) + '\n';
31
32
  const parseErrorLine = stringifyJson({ jsonrpc, error: parseError, id: null }) + '\n';
33
+ const internalErrorLine = (id) => stringifyJson({ jsonrpc, error: internalError, id }) + '\n';
32
34
  const ping = (id) => `{"jsonrpc":"2.0","method":"ping","id":${id}}`;
35
+ // One byte past `maxLengthBytes` on its own; embedded in a response envelope
36
+ // it stays comfortably over the limit despite the surrounding JSON overhead.
37
+ const oversizedString = 'a'.repeat(Number(maxLengthBytes) + 1);
33
38
  const notification = '{"jsonrpc":"2.0","method":"notifications/initialized"}';
34
39
  export const proof = {
35
40
  // EOF on the very first read: clean shutdown, nothing written, no further reads.
@@ -81,6 +86,52 @@ export const proof = {
81
86
  const state = runStep(step)(ping(1) + '\n');
82
87
  assertEq(state.stdout, okResponse(1));
83
88
  },
89
+ // A response that would exceed `maxLengthBytes` once UTF-8 encoded cannot
90
+ // be written as a single bit vector (`tryUtf8` reports overflow); the loop
91
+ // writes a JSON-RPC internal-error response — carrying the original
92
+ // request's `id`, not `null` — instead of throwing or silently dropping
93
+ // the reply.
94
+ oversizedResponseWritesInternalError: () => {
95
+ const step = (value) => {
96
+ const id = idOf(value);
97
+ return pure(id === undefined
98
+ ? null
99
+ : { jsonrpc, result: { big: oversizedString }, id });
100
+ };
101
+ const state = runStep(step)(ping(1) + '\n');
102
+ assertEq(state.stdout, internalErrorLine(1));
103
+ },
104
+ // The loop recovers from the oversized-response error and keeps draining
105
+ // stdin: a well-behaved request on the next line still gets its normal
106
+ // reply.
107
+ loopContinuesAfterOversizedResponse: () => {
108
+ const step = (value) => {
109
+ const id = idOf(value);
110
+ return pure(id === undefined
111
+ ? null
112
+ : id === 1
113
+ ? { jsonrpc, result: { big: oversizedString }, id }
114
+ : { jsonrpc, result: { ok: true }, id });
115
+ };
116
+ const state = runStep(step)([ping(1), ping(2)].join('\n'));
117
+ assertEq(state.stdout, internalErrorLine(1) + okResponse(2));
118
+ assertEq(state.stdin.length, 0);
119
+ },
120
+ // When even the `id`-preserving internal-error fallback would overflow —
121
+ // because the `id` itself is the oversized part, not just `result` — the
122
+ // loop falls back once more to a fixed `id: null` internal-error, the only
123
+ // shape in this transport guaranteed to always fit. Without this second
124
+ // fallback tier the request would get no response line at all.
125
+ oversizedIdFallsBackToNullId: () => {
126
+ const step = (value) => {
127
+ const id = idOf(value);
128
+ return pure(id === undefined
129
+ ? null
130
+ : { jsonrpc, result: { ok: true }, id: oversizedString });
131
+ };
132
+ const state = runStep(step)(ping(1) + '\n');
133
+ assertEq(state.stdout, internalErrorLine(null));
134
+ },
84
135
  // A multi-line session interleaving all cases: request, notification, and
85
136
  // malformed line, ending with an unterminated request. Order is preserved
86
137
  // and the notification contributes nothing.
@@ -7,19 +7,29 @@
7
7
  */
8
8
  import { type Vec } from '../types/bit_vec/module.f.ts';
9
9
  import { type List } from '../types/list/module.f.ts';
10
+ import { type Nullable } from '../types/nullable/module.f.ts';
10
11
  export type Block = ItemThunk | ItemArray;
11
12
  type ItemArray = readonly Item[];
12
13
  type ItemThunk = () => List<Item>;
13
14
  export type Item = string | ItemArray | ItemThunk;
14
15
  export declare const flat: (indent: string) => (text: Block) => List<string>;
15
16
  export type Utf8 = Vec;
17
+ /**
18
+ * Converts a string to an UTF-8, represented as an MSB first bit vector,
19
+ * returning `null` instead of throwing if the result would exceed
20
+ * `maxLength`.
21
+ *
22
+ * @param s The input string to be converted.
23
+ * @returns The resulting UTF-8 bit vector, MSB first, or `null` on overflow.
24
+ */
25
+ export declare const tryUtf8: (s: string) => Nullable<Utf8>;
16
26
  /**
17
27
  * Converts a string to an UTF-8, represented as an MSB first bit vector.
18
28
  *
19
29
  * @param s The input string to be converted.
20
30
  * @returns The resulting UTF-8 bit vector, MSB first.
21
31
  */
22
- export declare const utf8: (s: string) => Utf8;
32
+ export declare const utf8: import("../types/function/module.f.ts").Func<string, Vec>;
23
33
  /**
24
34
  * Converts a UTF-8 bit vector with MSB first encoding to a string.
25
35
  *
@@ -5,10 +5,11 @@
5
5
  *
6
6
  * @module
7
7
  */
8
- import { msb, u8List, u8ListToVec } from "../types/bit_vec/module.f.js";
8
+ import { msb, tryU8ListToVec, u8List } from "../types/bit_vec/module.f.js";
9
9
  import { flatMap } from "../types/list/module.f.js";
10
10
  import { fromCodePointList, toCodePointList } from "./utf8/module.f.js";
11
11
  import { stringToCodePointList, codePointListToString } from "./utf16/module.f.js";
12
+ import { mapUnwrap } from "../types/nullable/module.f.js";
12
13
  export const flat = (indent) => {
13
14
  const f = (prefix) => {
14
15
  const g = (item) => typeof (item) === 'string' ? [`${prefix}${item}`] : f(`${prefix}${indent}`)(item);
@@ -16,14 +17,23 @@ export const flat = (indent) => {
16
17
  };
17
18
  return f('');
18
19
  };
19
- const u8ListToVecMsb = u8ListToVec(msb);
20
+ const tryU8ListToVecMsb = tryU8ListToVec(msb);
21
+ /**
22
+ * Converts a string to an UTF-8, represented as an MSB first bit vector,
23
+ * returning `null` instead of throwing if the result would exceed
24
+ * `maxLength`.
25
+ *
26
+ * @param s The input string to be converted.
27
+ * @returns The resulting UTF-8 bit vector, MSB first, or `null` on overflow.
28
+ */
29
+ export const tryUtf8 = (s) => tryU8ListToVecMsb(fromCodePointList(stringToCodePointList(s)));
20
30
  /**
21
31
  * Converts a string to an UTF-8, represented as an MSB first bit vector.
22
32
  *
23
33
  * @param s The input string to be converted.
24
34
  * @returns The resulting UTF-8 bit vector, MSB first.
25
35
  */
26
- export const utf8 = (s) => u8ListToVecMsb(fromCodePointList(stringToCodePointList(s)));
36
+ export const utf8 = mapUnwrap(tryUtf8);
27
37
  /**
28
38
  * Converts a UTF-8 bit vector with MSB first encoding to a string.
29
39
  *
@@ -1,4 +1,10 @@
1
1
  export declare const proof: {
2
2
  block: () => void;
3
3
  encoding: () => void;
4
+ tryUtf8RoundTrip: () => void;
5
+ tryUtf8Empty: () => void;
6
+ tryUtf8Overflow: {
7
+ try: () => void;
8
+ throw: () => void;
9
+ };
4
10
  };
@@ -1,5 +1,8 @@
1
- import { flat, utf8, utf8ToString } from "./module.f.js";
1
+ import { assert, assertEq } from "../asserts/module.f.js";
2
+ import { flat, utf8, utf8ToString, tryUtf8 } from "./module.f.js";
2
3
  import { join } from "../types/string/module.f.js";
4
+ import { empty, maxLengthBytes } from "../types/bit_vec/module.f.js";
5
+ const overflowStr = 'a'.repeat(Number(maxLengthBytes) + 1);
3
6
  export const proof = {
4
7
  block: () => {
5
8
  const text = [
@@ -22,5 +25,25 @@ export const proof = {
22
25
  if (r !== 'Hello world!') {
23
26
  throw r;
24
27
  }
25
- }
28
+ },
29
+ tryUtf8RoundTrip: () => {
30
+ const v = tryUtf8('Hello world!');
31
+ assert(v !== null);
32
+ assertEq(v, utf8('Hello world!'));
33
+ assertEq(utf8ToString(v), 'Hello world!');
34
+ },
35
+ tryUtf8Empty: () => {
36
+ assertEq(tryUtf8(''), empty);
37
+ },
38
+ tryUtf8Overflow: {
39
+ // One byte past `maxLengthBytes`; `tryUtf8` should report `null`
40
+ // instead of building an oversized `bigint`, and the throwing `utf8`
41
+ // wrapper should raise on the same input.
42
+ try: () => {
43
+ assertEq(tryUtf8(overflowStr), null);
44
+ },
45
+ throw: () => {
46
+ utf8(overflowStr);
47
+ },
48
+ },
26
49
  };
@@ -26,6 +26,7 @@ import type { Binary, Fold, Reduce as OpReduce } from '../function/operator/modu
26
26
  import { type List, type Thunk } from '../list/module.f.ts';
27
27
  import { type Nominal } from '../nominal/module.f.ts';
28
28
  import { type Sign } from '../function/compare/module.f.ts';
29
+ import { type Nullable } from '../nullable/module.f.ts';
29
30
  /**
30
31
  * A vector of bits represented as a signed `bigint`.
31
32
  */
@@ -195,6 +196,12 @@ export type BitOrder = {
195
196
  * ```
196
197
  */
197
198
  readonly concat: Reduce;
199
+ /**
200
+ * Folds a list of vectors into a single vector in this bit order, like
201
+ * `listToVec`, but returns `null` instead of throwing when the combined
202
+ * length would exceed `maxLength`.
203
+ */
204
+ readonly tryListToVec: (list: List<Vec>) => Nullable<Vec>;
198
205
  /**
199
206
  * Folds a list of vectors into a single vector in this bit order.
200
207
  *
@@ -221,9 +228,10 @@ export type BitOrder = {
221
228
  */
222
229
  readonly cmp: (a: Vec) => (b: Vec) => Sign;
223
230
  readonly unpackSplit: (len: bigint) => (u: Unpacked) => readonly [bigint, bigint];
224
- readonly unpackConcat: (a: Unpacked) => (b: Unpacked) => Unpacked;
231
+ readonly unpackConcat: UnpackConcat;
225
232
  readonly startsWith: (prefix: Vec) => (v: Vec) => boolean;
226
233
  };
234
+ type UnpackConcat = (a: Unpacked) => (b: Unpacked) => Unpacked;
227
235
  /**
228
236
  * Implements operations for handling vectors in a least-significant-bit (LSb) first order.
229
237
  *
@@ -240,6 +248,16 @@ export declare const lsb: BitOrder;
240
248
  * Usually associated with Big-Endian (BE) byte order.
241
249
  */
242
250
  export declare const msb: BitOrder;
251
+ /**
252
+ * Converts a list of unsigned 8-bit integers to a bit vector using the provided
253
+ * bit order, like `u8ListToVec`, but returns `null` instead of throwing when the
254
+ * result would exceed `maxLength`.
255
+ *
256
+ * @param bo The bit order for the conversion
257
+ * @param list The list of unsigned 8-bit integers to be converted.
258
+ * @returns The resulting vector, or `null` if it would exceed `maxLength`.
259
+ */
260
+ export declare const tryU8ListToVec: ({ unpackConcat }: BitOrder) => (list: List<number>) => Nullable<Vec>;
243
261
  /**
244
262
  * Converts a list of unsigned 8-bit integers to a bit vector using the provided bit order.
245
263
  *
@@ -247,7 +265,7 @@ export declare const msb: BitOrder;
247
265
  * @param list The list of unsigned 8-bit integers to be converted.
248
266
  * @returns The resulting vector based on the provided bit order.
249
267
  */
250
- export declare const u8ListToVec: ({ unpackConcat }: BitOrder) => (list: List<number>) => Vec;
268
+ export declare const u8ListToVec: (bo: BitOrder) => import("../function/module.f.ts").Func<List<number>, Vec>;
251
269
  /**
252
270
  * Chunks an unpacked vector into fixed-size pieces of `n` bits using the provided bit order,
253
271
  * returning each chunk as an unsigned integer.
@@ -23,10 +23,11 @@
23
23
  */
24
24
  import { bitLength, divUp, mask, maxLength, xor } from "../bigint/module.f.js";
25
25
  import { flip, identity } from "../function/module.f.js";
26
- import { fold, iterable, map } from "../list/module.f.js";
26
+ import { iterable, map } from "../list/module.f.js";
27
27
  import { asBase, asNominal } from "../nominal/module.f.js";
28
28
  import { repeat as mRepeat } from "../monoid/module.f.js";
29
29
  import { cmp, max, min } from "../function/compare/module.f.js";
30
+ import { mapUnwrap } from "../nullable/module.f.js";
30
31
  /**
31
32
  * Maximum length of a bit vector in bits (1_048_576 = 0x10_0000).
32
33
  * This limit is enforced by Bun's `bigint` size constraint, the minimal limit
@@ -133,6 +134,64 @@ const op = (norm) => (op) => ap => bp => {
133
134
  const { a, b } = norm(au)(bu)(len);
134
135
  return vec(len)(op(a)(b));
135
136
  };
137
+ const unpackEmpty = { length: 0n, uint: 0n };
138
+ const listToVecOp = (unpackConcat) => ({
139
+ init: { len: 0n, stack: [] },
140
+ update: (v, { len, stack }) => {
141
+ len += v.length;
142
+ if (len > maxLength) {
143
+ return null;
144
+ }
145
+ let i = 0;
146
+ while (true) {
147
+ if (stack.length <= i) {
148
+ stack = [...stack, v];
149
+ break;
150
+ }
151
+ const old = stack[i];
152
+ if (old.length === 0n) {
153
+ stack = stack.toSpliced(i, 1, v);
154
+ break;
155
+ }
156
+ stack = stack.toSpliced(i, 1, unpackEmpty);
157
+ v = unpackConcat(old)(v);
158
+ i++;
159
+ }
160
+ return { len, stack };
161
+ },
162
+ end: ({ stack }) => pack(stack.reduce((p, c) => unpackConcat(c)(p), unpackEmpty))
163
+ });
164
+ /**
165
+ * Concatenates a list of unpacked vectors using a binary-counter accumulator,
166
+ * giving O(n log n) total `bigint` shifting work instead of the O(n²) of a
167
+ * naive left fold.
168
+ *
169
+ * Slot `i` of `result` holds an already-combined run of the most recent
170
+ * `2 ** i` elements. Each arriving element "carries" upward, merging only with
171
+ * runs of comparable size — exactly like incrementing a binary number — so
172
+ * every merge joins two runs of similar length. Left-to-right element order is
173
+ * preserved: `unpackConcat(old)(cur)` keeps the earlier run on the left, and
174
+ * the final reduce prepends higher (earlier) slots in front of accumulated
175
+ * later runs. An empty list yields `unpackEmpty`.
176
+ *
177
+ * This is the bit-vector analogue of a builder that accumulates appended pieces
178
+ * and materializes the combined result on demand, such as `StringBuilder`
179
+ * (Java, C#) or `strings.Builder` (Go).
180
+ */
181
+ const unpackListToVec = (unpackConcat) => {
182
+ const { init, update, end } = listToVecOp(unpackConcat);
183
+ return (list) => {
184
+ let result = init;
185
+ for (const e of iterable(list)) {
186
+ const candidate = update(e, result);
187
+ if (candidate === null) {
188
+ return null;
189
+ }
190
+ result = candidate;
191
+ }
192
+ return end(result);
193
+ };
194
+ };
136
195
  const bo = ({ front, removeFront, norm, uintCmp, unpackSplit, unpackConcatUint }) => {
137
196
  const unpackPopFront = (len) => {
138
197
  const m = mask(len);
@@ -142,8 +201,9 @@ const bo = ({ front, removeFront, norm, uintCmp, unpackSplit, unpackConcatUint }
142
201
  return [uint & m, { length: v.length - len, uint: rest }];
143
202
  };
144
203
  };
145
- const unpackConcat = (a) => (b) => ({
146
- length: a.length + b.length, uint: unpackConcatUint(a)(b)
204
+ const unpackConcat = a => b => ({
205
+ length: a.length + b.length,
206
+ uint: unpackConcatUint(a)(b)
147
207
  });
148
208
  const popFront = len => {
149
209
  const f = unpackPopFront(len);
@@ -157,11 +217,13 @@ const bo = ({ front, removeFront, norm, uintCmp, unpackSplit, unpackConcatUint }
157
217
  const bu = unpack(b);
158
218
  return pack(unpackConcat(au)(bu));
159
219
  };
220
+ const tryListToVec = (list) => unpackListToVec(unpackConcat)(map(unpack)(list));
160
221
  return {
161
222
  front,
162
223
  removeFront,
163
224
  concat,
164
- listToVec: fold(flip(concat))(empty),
225
+ tryListToVec,
226
+ listToVec: mapUnwrap(tryListToVec),
165
227
  xor: op(norm)(xor),
166
228
  unpackPopFront,
167
229
  popFront,
@@ -232,7 +294,16 @@ export const msb = bo({
232
294
  unpackSplit: len => ({ length, uint }) => [uint >> (length - len), uint],
233
295
  unpackConcatUint: flip(lsbUnpackConcatUint),
234
296
  });
235
- const unpackEmpty = { length: 0n, uint: 0n };
297
+ /**
298
+ * Converts a list of unsigned 8-bit integers to a bit vector using the provided
299
+ * bit order, like `u8ListToVec`, but returns `null` instead of throwing when the
300
+ * result would exceed `maxLength`.
301
+ *
302
+ * @param bo The bit order for the conversion
303
+ * @param list The list of unsigned 8-bit integers to be converted.
304
+ * @returns The resulting vector, or `null` if it would exceed `maxLength`.
305
+ */
306
+ export const tryU8ListToVec = ({ unpackConcat }) => (list) => unpackListToVec(unpackConcat)(map((b) => ({ length: 8n, uint: BigInt(b) }))(list));
236
307
  /**
237
308
  * Converts a list of unsigned 8-bit integers to a bit vector using the provided bit order.
238
309
  *
@@ -240,28 +311,7 @@ const unpackEmpty = { length: 0n, uint: 0n };
240
311
  * @param list The list of unsigned 8-bit integers to be converted.
241
312
  * @returns The resulting vector based on the provided bit order.
242
313
  */
243
- export const u8ListToVec = ({ unpackConcat }) => (list) => {
244
- let result = [];
245
- for (const b of iterable(list)) {
246
- let v = { length: 8n, uint: BigInt(b) };
247
- let i = 0;
248
- while (true) {
249
- if (result.length <= i) {
250
- result = [...result, v];
251
- break;
252
- }
253
- const old = result[i];
254
- if (old.length === 0n) {
255
- result = result.toSpliced(i, 1, v);
256
- break;
257
- }
258
- result = result.toSpliced(i, 1, unpackEmpty);
259
- v = unpackConcat(old)(v);
260
- i++;
261
- }
262
- }
263
- return pack(result.reduce((p, c) => unpackConcat(c)(p), unpackEmpty));
264
- };
314
+ export const u8ListToVec = (bo) => mapUnwrap(tryU8ListToVec(bo));
265
315
  const unpackChunkList = ({ unpackSplit }) => (n) => {
266
316
  const divUpN2 = divUp(n << 1n);
267
317
  return u => {
@@ -46,6 +46,14 @@ export declare const proof: {
46
46
  emptyVec: () => void;
47
47
  };
48
48
  u8ListToVec: () => () => void;
49
+ tryListToVecOverflow: () => void;
50
+ listToVecOverflow: {
51
+ throw: () => void;
52
+ };
53
+ u8ListToVecOverflow: {
54
+ try: () => void;
55
+ throw: () => void;
56
+ };
49
57
  u8ListUnaligned: () => void;
50
58
  chunkList: {
51
59
  empty: () => void;
@@ -1,6 +1,7 @@
1
+ import { assertEq } from "../../asserts/module.f.js";
1
2
  import { mask } from "../bigint/module.f.js";
2
3
  import { asBase, asNominal } from "../nominal/module.f.js";
3
- import { length, empty, uint, vec, lsb, msb, repeat, vec8, u8ListToVec, u8List, chunkList, fromSentinel } from "./module.f.js";
4
+ import { length, empty, uint, vec, lsb, msb, repeat, vec8, maxLength, u8ListToVec, tryU8ListToVec, u8List, chunkList, fromSentinel } from "./module.f.js";
4
5
  import { repeat as listRepeat, toArray } from "../list/module.f.js";
5
6
  const unsafeVec = (a) => asNominal(a);
6
7
  // 0x8 = 0b1000 = 0 + 8
@@ -11,11 +12,6 @@ const unsafeVec = (a) => asNominal(a);
11
12
  // 0xD = 0b1101 = 5 + 8
12
13
  // 0xE = 0b1110 = 6 + 8
13
14
  // 0xF = 0b1111 = 7 + 8
14
- const assertEq = (a, b) => {
15
- if (a !== b) {
16
- throw [a, b];
17
- }
18
- };
19
15
  const assertEq2 = ([a0, a1], [b0, b1]) => {
20
16
  assertEq(a0, b0);
21
17
  assertEq(a1, b1);
@@ -504,6 +500,28 @@ export const proof = {
504
500
  }
505
501
  };
506
502
  },
503
+ tryListToVecOverflow: () => {
504
+ const list = [vec(maxLength)(1n), vec(1n)(1n)];
505
+ assertEq(lsb.tryListToVec(list), null);
506
+ assertEq(msb.tryListToVec(list), null);
507
+ },
508
+ listToVecOverflow: {
509
+ // Same oversized input, but through the throwing `listToVec` wrapper.
510
+ throw: () => {
511
+ const list = { first: vec(maxLength + 1n)(1n), tail: null };
512
+ lsb.listToVec(list);
513
+ },
514
+ },
515
+ u8ListToVecOverflow: {
516
+ // 131_073 bytes is 8 bits past `maxLength`; same null/throw split as
517
+ // `tryListToVec`/`listToVec` above, exercised through the byte-list API.
518
+ try: () => {
519
+ assertEq(tryU8ListToVec(msb)(listRepeat(0x12)(131_073)), null);
520
+ },
521
+ throw: () => {
522
+ u8ListToVec(msb)(listRepeat(0x12)(131_073));
523
+ },
524
+ },
507
525
  u8ListUnaligned: () => {
508
526
  const x = vec(9n)(0x83n);
509
527
  const a = toArray(u8List(msb)(x));
@@ -1,8 +1,3 @@
1
- /**
2
- * Utilities for nullable (`null`/`undefined`) value handling.
3
- *
4
- * @module
5
- */
6
1
  import type { Option } from '../option/module.f.ts';
7
2
  export type Nullable<T> = T | null;
8
3
  export declare const map: <T, R>(f: (value: T) => R) => (value: Nullable<T>) => Nullable<R>;
@@ -15,3 +10,12 @@ export declare const toOption: <T>(value: Nullable<T>) => Option<T>;
15
10
  * property/index lookups) and FunctionalScript (which uses `null` for absence).
16
11
  */
17
12
  export declare const fromUndefined: <T>(value: T | undefined) => Nullable<T>;
13
+ /**
14
+ * Extracts the value from a `Nullable`, asserting that it is not `null`.
15
+ */
16
+ export declare const unwrap: <T>(value: Nullable<T>) => T;
17
+ /**
18
+ * Lifts a function that signals failure with `null` into one that asserts
19
+ * success instead, unwrapping the result.
20
+ */
21
+ export declare const mapUnwrap: <I, T>(f: (i: I) => Nullable<T>) => import("../function/module.f.ts").Func<I, T>;
@@ -1,3 +1,10 @@
1
+ /**
2
+ * Utilities for nullable (`null`/`undefined`) value handling.
3
+ *
4
+ * @module
5
+ */
6
+ import { assert } from "../../asserts/module.f.js";
7
+ import { fn } from "../function/module.f.js";
1
8
  export const map = f => value => value === null ? null : f(value);
2
9
  export const match = f => none => value => value === null ? none() : f(value);
3
10
  export const toOption = (value) => value === null ? [] : [value];
@@ -8,3 +15,15 @@ export const toOption = (value) => value === null ? [] : [value];
8
15
  * property/index lookups) and FunctionalScript (which uses `null` for absence).
9
16
  */
10
17
  export const fromUndefined = (value) => value === undefined ? null : value;
18
+ /**
19
+ * Extracts the value from a `Nullable`, asserting that it is not `null`.
20
+ */
21
+ export const unwrap = (value) => {
22
+ assert(value !== null);
23
+ return value;
24
+ };
25
+ /**
26
+ * Lifts a function that signals failure with `null` into one that asserts
27
+ * success instead, unwrapping the result.
28
+ */
29
+ export const mapUnwrap = (f) => fn(f).map(unwrap).result;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "functionalscript",
3
- "version": "0.34.0",
3
+ "version": "0.35.0",
4
4
  "type": "module",
5
5
  "files": [
6
6
  "**/*.js",
@@ -44,7 +44,7 @@
44
44
  "homepage": "https://github.com/functionalscript/functionalscript#readme",
45
45
  "devDependencies": {
46
46
  "@playwright/test": "1.61.1",
47
- "@types/node": "26.0.1",
47
+ "@types/node": "26.1.0",
48
48
  "typescript": "6.0.3"
49
49
  }
50
50
  }