@statewalker/webrun-msgpack 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022-2026 statewalker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,137 @@
+# @statewalker/webrun-msgpack
+
+Length-prefixed MessagePack frame codec for async iterables. Streams-safe `encode`/`decode` for arbitrary values, plus zero-copy specialisations for `Float32Array`.
+
+## Why it exists
+
+Consumers that pipe values across transports (scanners writing chunks to a store, chat pipelines streaming embeddings, etc.) need a way to serialise a stream of objects into a byte stream and reassemble it on the other side without truncation surprises.
+
+A raw MessagePack stream has no frame boundaries: a decoder can only succeed if the chunk boundaries happen to line up with the payload boundaries. Length-prefix framing fixes this — the decoder buffers incoming bytes and only yields when a complete `[length][payload]` pair is available. Partial trailing frames are NEVER emitted, so callers can detect truncation by comparing observed count to expected.
+
+Previously the codec lived inside `@repo/streams` (private, unpublished). It's been extracted here so (a) consumers that only need framing don't pull in the broader `webrun-streams` surface, and (b) the `@ygoe/msgpack` dependency lives in exactly one place.
+
+## How to use
+
+```sh
+npm install @statewalker/webrun-msgpack
+```
+
+Four exports — one encode/decode pair for generic values, one for `Float32Array`:
+
+| Export | Direction | Use case |
+| --- | --- | --- |
+| `encodeMsgpack<T>(src: AsyncIterable<T>)` | values → bytes | generic JSON-ish values |
+| `decodeMsgpack<T>(src: AsyncIterable<Uint8Array>)` | bytes → values | inverse of `encodeMsgpack` |
+| `encodeFloat32Arrays(src: AsyncIterable<Float32Array>)` | arrays → bytes | zero-copy float streaming |
+| `decodeFloat32Arrays(src: AsyncIterable<Uint8Array>)` | bytes → arrays | inverse of `encodeFloat32Arrays` |
+
+## Examples
+
+### Stream of values
+
+```ts
+import { encodeMsgpack, decodeMsgpack } from "@statewalker/webrun-msgpack";
+
+async function* events() {
+  yield { type: "start" };
+  yield { type: "chunk", text: "hello" };
+  yield { type: "done" };
+}
+
+// encode
+const bytes: AsyncIterable<Uint8Array> = encodeMsgpack(events());
+
+// decode on the other side — handles arbitrary chunk boundaries
+for await (const msg of decodeMsgpack<{ type: string; text?: string }>(bytes)) {
+  console.log(msg);
+}
+```
+
+### Embeddings pipeline
+
+```ts
+import {
+  decodeFloat32Arrays,
+  encodeFloat32Arrays,
+} from "@statewalker/webrun-msgpack";
+
+async function* chunks() {
+  yield new Float32Array([0.1, 0.2, 0.3, 0.4]);
+  yield new Float32Array([0.5, 0.6, 0.7, 0.8]);
+}
+
+// wire-efficient: msgpack `bin` type reinterpreted byte-for-byte as Float32.
+const pipe = decodeFloat32Arrays(encodeFloat32Arrays(chunks()));
+for await (const arr of pipe) console.log(arr.length); // 4, 4
+```
+
+### Re-framing across transport chunks
+
+```ts
+import { decodeMsgpack, encodeMsgpack } from "@statewalker/webrun-msgpack";
+
+// Produce one frame, then split the bytes any way you like:
+const bytes = [];
+for await (const f of encodeMsgpack([{ a: 1, b: "hi" }])) bytes.push(f);
+// Hand the decoder arbitrarily small slices — it buffers until complete:
+async function* byOne() {
+  for (const b of bytes) for (const byte of b) yield new Uint8Array([byte]);
+}
+for await (const v of decodeMsgpack<{ a: number; b: string }>(byOne())) {
+  console.log(v); // { a: 1, b: "hi" }
+}
+```
+
+## Internals
+
+### Frame layout
+
+```
+┌──────────────┬────────────────────────────┐
+│  uint32 BE   │    msgpack payload         │
+│  (4 bytes)   │    (`length` bytes)        │
+└──────────────┴────────────────────────────┘
+```
+
+- Big-endian 32-bit length prefix — same convention as Java `DataOutputStream` and most wire protocols.
+- Max payload per frame: 2³²−1 bytes. No fragmentation within a frame (a single call to `serialize` produces the whole payload up-front); very large values will allocate proportionally.
+
+### Decoder state machine
+
+The decoder keeps a rolling `Uint8Array` buffer. Each incoming chunk is appended (single allocation per chunk); then:
+
+1. If buffer is shorter than 4 bytes — wait for more.
+2. Read the 32-bit BE length.
+3. If buffer doesn't hold `4 + length` bytes — wait for more.
+4. Slice the payload, deserialise with `@ygoe/msgpack`, yield.
+5. Advance the buffer past this frame; repeat step 1.
+
+Zero-length chunks are tolerated and simply no-op through the loop. Truncated trailing frames are silently dropped — the buffer retains them but the consuming `for await` ends without yielding a partial value.
+
+### Float32Array zero-copy
+
+`encodeFloat32Arrays` constructs a `Uint8Array` view over the `Float32Array`'s underlying buffer and serialises it as a msgpack `bin` payload — no float-by-float conversion. `decodeFloat32Arrays` reinterprets the decoded `Uint8Array` as a `Float32Array`. When the decoded buffer's `byteOffset` is not 4-byte aligned (can happen if `@ygoe/msgpack` returns a view into a larger buffer), we copy into a fresh aligned `Uint8Array` before constructing the `Float32Array`; otherwise the operation is view-only.
+
+### Dependencies
+
+- [`@ygoe/msgpack`](https://github.com/ygoe/msgpack.js) — single-file msgpack implementation (≈7 kB gzipped), no transitive deps.
+
+Dev: TypeScript, vitest, tsdown, rimraf (catalog versions from the monorepo root).
+
+### Constraints
+
+- Big-endian length prefix only — no little-endian variant.
+- `decodeMsgpack` allocates one `Uint8Array` per incoming chunk for the `concat`; long streams with many tiny chunks may benefit from a batched source upstream.
+- `Float32Array` codec is strictly `Float32` — no element-size negotiation.
+
+## Scripts
+
+```sh
+pnpm test        # vitest run (25 tests)
+pnpm run build   # tsdown
+pnpm lint        # biome check
+```
+
+## License
+
+MIT © statewalker

package/dist/index.d.mts

@@ -0,0 +1,23 @@
+//#region src/msgpack.d.ts
+/**
+ * Encode each value as a length-prefixed msgpack frame.
+ * Frame format: [4-byte big-endian length][msgpack bytes]
+ */
+declare function encodeMsgpack<T>(input: AsyncIterable<T>): AsyncGenerator<Uint8Array>;
+/**
+ * Decode length-prefixed msgpack frames, reassembling across chunk boundaries.
+ */
+declare function decodeMsgpack<T>(input: AsyncIterable<Uint8Array>): AsyncGenerator<T>;
+/**
+ * Encode each Float32Array as a msgpack frame.
+ * The Float32Array is converted to a Uint8Array view (zero-copy) before encoding as msgpack bin.
+ */
+declare function encodeFloat32Arrays(input: AsyncIterable<Float32Array>): AsyncGenerator<Uint8Array>;
+/**
+ * Decode msgpack frames back to Float32Array.
+ * Each frame contains a msgpack bin value (Uint8Array), reinterpreted as Float32Array.
+ */
+declare function decodeFloat32Arrays(input: AsyncIterable<Uint8Array>): AsyncGenerator<Float32Array>;
+//#endregion
+export { decodeFloat32Arrays, decodeMsgpack, encodeFloat32Arrays, encodeMsgpack };
+//# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/msgpack.ts"],"mappings":";;;;;iBAQuB,cAAc,GAAG,OAAO,cAAc,KAAK,eAAe;;;;iBAc1D,cAAc,GAAG,OAAO,cAAc,cAAc,eAAe;;;;;iBAsBnE,oBACrB,OAAO,cAAc,gBACpB,eAAe;;;;;iBAgBK,oBACrB,OAAO,cAAc,cACpB,eAAe"}

package/dist/index.mjs

@@ -0,0 +1,76 @@
+import msgpack from "@ygoe/msgpack";
+//#region src/msgpack.ts
+const { serialize, deserialize } = msgpack;
+/**
+* Encode each value as a length-prefixed msgpack frame.
+* Frame format: [4-byte big-endian length][msgpack bytes]
+*/
+async function* encodeMsgpack(input) {
+	for await (const item of input) {
+		const payload = serialize(item);
+		const frame = new Uint8Array(4 + payload.length);
+		new DataView(frame.buffer).setUint32(0, payload.length, false);
+		frame.set(payload, 4);
+		yield frame;
+	}
+}
+/**
+* Decode length-prefixed msgpack frames, reassembling across chunk boundaries.
+*/
+async function* decodeMsgpack(input) {
+	let buffer = /* @__PURE__ */ new Uint8Array(0);
+	for await (const chunk of input) {
+		buffer = concat(buffer, chunk);
+		while (buffer.length >= 4) {
+			const frameLen = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength).getUint32(0, false);
+			if (buffer.length < 4 + frameLen) break;
+			const payload = buffer.subarray(4, 4 + frameLen);
+			buffer = buffer.subarray(4 + frameLen);
+			yield deserialize(Uint8Array.from(payload));
+		}
+	}
+}
+/**
+* Encode each Float32Array as a msgpack frame.
+* The Float32Array is converted to a Uint8Array view (zero-copy) before encoding as msgpack bin.
+*/
+async function* encodeFloat32Arrays(input) {
+	for await (const arr of input) {
+		const bytes = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);
+		const payload = serialize(bytes);
+		const frame = new Uint8Array(4 + payload.length);
+		new DataView(frame.buffer).setUint32(0, payload.length, false);
+		frame.set(payload, 4);
+		yield frame;
+	}
+}
+/**
+* Decode msgpack frames back to Float32Array.
+* Each frame contains a msgpack bin value (Uint8Array), reinterpreted as Float32Array.
+*/
+async function* decodeFloat32Arrays(input) {
+	for await (const item of decodeMsgpack(wrapIterable(input))) {
+		const aligned = alignBuffer(item);
+		yield new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
+	}
+}
+function alignBuffer(bytes) {
+	if (bytes.byteOffset % 4 === 0) return bytes;
+	const aligned = new Uint8Array(bytes.length);
+	aligned.set(bytes);
+	return aligned;
+}
+function concat(a, b) {
+	if (a.length === 0) return b;
+	const result = new Uint8Array(a.length + b.length);
+	result.set(a, 0);
+	result.set(b, a.length);
+	return result;
+}
+async function* wrapIterable(input) {
+	yield* input;
+}
+//#endregion
+export { decodeFloat32Arrays, decodeMsgpack, encodeFloat32Arrays, encodeMsgpack };
+
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../src/msgpack.ts"],"sourcesContent":["import msgpack from \"@ygoe/msgpack\";\n\nconst { serialize, deserialize } = msgpack;\n\n/**\n * Encode each value as a length-prefixed msgpack frame.\n * Frame format: [4-byte big-endian length][msgpack bytes]\n */\nexport async function* encodeMsgpack<T>(input: AsyncIterable<T>): AsyncGenerator<Uint8Array> {\n  for await (const item of input) {\n    const payload = serialize(item);\n    const frame = new Uint8Array(4 + payload.length);\n    const view = new DataView(frame.buffer);\n    view.setUint32(0, payload.length, false);\n    frame.set(payload, 4);\n    yield frame;\n  }\n}\n\n/**\n * Decode length-prefixed msgpack frames, reassembling across chunk boundaries.\n */\nexport async function* decodeMsgpack<T>(input: AsyncIterable<Uint8Array>): AsyncGenerator<T> {\n  let buffer: Uint8Array = new Uint8Array(0);\n\n  for await (const chunk of input) {\n    buffer = concat(buffer, chunk);\n\n    while (buffer.length >= 4) {\n      const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);\n      const frameLen = view.getUint32(0, false);\n      if (buffer.length < 4 + frameLen) break;\n\n      const payload = buffer.subarray(4, 4 + frameLen);\n      buffer = buffer.subarray(4 + frameLen);\n      yield deserialize(Uint8Array.from(payload)) as T;\n    }\n  }\n}\n\n/**\n * Encode each Float32Array as a msgpack frame.\n * The Float32Array is converted to a Uint8Array view (zero-copy) before encoding as msgpack bin.\n */\nexport async function* encodeFloat32Arrays(\n  input: AsyncIterable<Float32Array>,\n): AsyncGenerator<Uint8Array> {\n  for await (const arr of input) {\n    const bytes = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);\n    const payload = serialize(bytes);\n    const frame = new Uint8Array(4 + payload.length);\n    const view = new DataView(frame.buffer);\n    view.setUint32(0, payload.length, false);\n    frame.set(payload, 4);\n    yield frame;\n  }\n}\n\n/**\n * Decode msgpack frames back to Float32Array.\n * Each frame contains a msgpack bin value (Uint8Array), reinterpreted as Float32Array.\n */\nexport async function* decodeFloat32Arrays(\n  input: AsyncIterable<Uint8Array>,\n): AsyncGenerator<Float32Array> {\n  for await (const item of decodeMsgpack<Uint8Array>(wrapIterable(input))) {\n    const aligned = alignBuffer(item);\n    yield new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);\n  }\n}\n\nfunction alignBuffer(bytes: Uint8Array): Uint8Array {\n  if (bytes.byteOffset % 4 === 0) return bytes;\n  const aligned = new Uint8Array(bytes.length);\n  aligned.set(bytes);\n  return aligned;\n}\n\nfunction concat(a: Uint8Array, b: Uint8Array): Uint8Array {\n  if (a.length === 0) return b;\n  const result = new Uint8Array(a.length + b.length);\n  result.set(a, 0);\n  result.set(b, a.length);\n  return result;\n}\n\nasync function* wrapIterable<T>(input: AsyncIterable<T>): AsyncGenerator<T> {\n  yield* input;\n}\n"],"mappings":";;AAEA,MAAM,EAAE,WAAW,gBAAgB;;;;;AAMnC,gBAAuB,cAAiB,OAAqD;CAC3F,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,UAAU,UAAU,IAAI;EAC9B,MAAM,QAAQ,IAAI,WAAW,IAAI,QAAQ,MAAM;EAE/C,IADiB,SAAS,MAAM,MAC7B,CAAC,CAAC,UAAU,GAAG,QAAQ,QAAQ,KAAK;EACvC,MAAM,IAAI,SAAS,CAAC;EACpB,MAAM;CACR;AACF;;;;AAKA,gBAAuB,cAAiB,OAAqD;CAC3F,IAAI,yBAAqB,IAAI,WAAW,CAAC;CAEzC,WAAW,MAAM,SAAS,OAAO;EAC/B,SAAS,OAAO,QAAQ,KAAK;EAE7B,OAAO,OAAO,UAAU,GAAG;GAEzB,MAAM,WAAW,IADA,SAAS,OAAO,QAAQ,OAAO,YAAY,OAAO,UAC/C,CAAC,CAAC,UAAU,GAAG,KAAK;GACxC,IAAI,OAAO,SAAS,IAAI,UAAU;GAElC,MAAM,UAAU,OAAO,SAAS,GAAG,IAAI,QAAQ;GAC/C,SAAS,OAAO,SAAS,IAAI,QAAQ;GACrC,MAAM,YAAY,WAAW,KAAK,OAAO,CAAC;EAC5C;CACF;AACF;;;;;AAMA,gBAAuB,oBACrB,OAC4B;CAC5B,WAAW,MAAM,OAAO,OAAO;EAC7B,MAAM,QAAQ,IAAI,WAAW,IAAI,QAAQ,IAAI,YAAY,IAAI,UAAU;EACvE,MAAM,UAAU,UAAU,KAAK;EAC/B,MAAM,QAAQ,IAAI,WAAW,IAAI,QAAQ,MAAM;EAE/C,IADiB,SAAS,MAAM,MAC7B,CAAC,CAAC,UAAU,GAAG,QAAQ,QAAQ,KAAK;EACvC,MAAM,IAAI,SAAS,CAAC;EACpB,MAAM;CACR;AACF;;;;;AAMA,gBAAuB,oBACrB,OAC8B;CAC9B,WAAW,MAAM,QAAQ,cAA0B,aAAa,KAAK,CAAC,GAAG;EACvE,MAAM,UAAU,YAAY,IAAI;EAChC,MAAM,IAAI,aAAa,QAAQ,QAAQ,QAAQ,YAAY,QAAQ,aAAa,CAAC;CACnF;AACF;AAEA,SAAS,YAAY,OAA+B;CAClD,IAAI,MAAM,aAAa,MAAM,GAAG,OAAO;CACvC,MAAM,UAAU,IAAI,WAAW,MAAM,MAAM;CAC3C,QAAQ,IAAI,KAAK;CACjB,OAAO;AACT;AAEA,SAAS,OAAO,GAAe,GAA2B;CACxD,IAAI,EAAE,WAAW,GAAG,OAAO;CAC3B,MAAM,SAAS,IAAI,WAAW,EAAE,SAAS,EAAE,MAAM;CACjD,OAAO,IAAI,GAAG,CAAC;CACf,OAAO,IAAI,GAAG,EAAE,MAAM;CACtB,OAAO;AACT;AAEA,gBAAgB,aAAgB,OAA4C;CAC1E,OAAO;AACT"}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@statewalker/webrun-msgpack",
+  "version": "0.1.1",
+  "private": false,
+  "type": "module",
+  "description": "Length-prefixed MessagePack frame codec for async iterables. Streams-safe encode/decode for values and Float32Arrays.",
+  "homepage": "https://github.com/statewalker/webrun-wire",
+  "author": {
+    "name": "Mikhail Kotelnikov",
+    "email": "mikhail.kotelnikov@gmail.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:statewalker/webrun-wire.git",
+    "directory": "packages/webrun-msgpack"
+  },
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@ygoe/msgpack": "^1.0.3"
+  },
+  "devDependencies": {
+    "rimraf": "^6.1.3",
+    "tsdown": "^0.22.14",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "lint": "biome check --write .",
+    "format": "biome format --write ."
+  }
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export {
+  decodeFloat32Arrays,
+  decodeMsgpack,
+  encodeFloat32Arrays,
+  encodeMsgpack,
+} from "./msgpack.js";

package/src/msgpack.ts

@@ -0,0 +1,89 @@
+import msgpack from "@ygoe/msgpack";
+
+const { serialize, deserialize } = msgpack;
+
+/**
+ * Encode each value as a length-prefixed msgpack frame.
+ * Frame format: [4-byte big-endian length][msgpack bytes]
+ */
+export async function* encodeMsgpack<T>(input: AsyncIterable<T>): AsyncGenerator<Uint8Array> {
+  for await (const item of input) {
+    const payload = serialize(item);
+    const frame = new Uint8Array(4 + payload.length);
+    const view = new DataView(frame.buffer);
+    view.setUint32(0, payload.length, false);
+    frame.set(payload, 4);
+    yield frame;
+  }
+}
+
+/**
+ * Decode length-prefixed msgpack frames, reassembling across chunk boundaries.
+ */
+export async function* decodeMsgpack<T>(input: AsyncIterable<Uint8Array>): AsyncGenerator<T> {
+  let buffer: Uint8Array = new Uint8Array(0);
+
+  for await (const chunk of input) {
+    buffer = concat(buffer, chunk);
+
+    while (buffer.length >= 4) {
+      const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+      const frameLen = view.getUint32(0, false);
+      if (buffer.length < 4 + frameLen) break;
+
+      const payload = buffer.subarray(4, 4 + frameLen);
+      buffer = buffer.subarray(4 + frameLen);
+      yield deserialize(Uint8Array.from(payload)) as T;
+    }
+  }
+}
+
+/**
+ * Encode each Float32Array as a msgpack frame.
+ * The Float32Array is converted to a Uint8Array view (zero-copy) before encoding as msgpack bin.
+ */
+export async function* encodeFloat32Arrays(
+  input: AsyncIterable<Float32Array>,
+): AsyncGenerator<Uint8Array> {
+  for await (const arr of input) {
+    const bytes = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);
+    const payload = serialize(bytes);
+    const frame = new Uint8Array(4 + payload.length);
+    const view = new DataView(frame.buffer);
+    view.setUint32(0, payload.length, false);
+    frame.set(payload, 4);
+    yield frame;
+  }
+}
+
+/**
+ * Decode msgpack frames back to Float32Array.
+ * Each frame contains a msgpack bin value (Uint8Array), reinterpreted as Float32Array.
+ */
+export async function* decodeFloat32Arrays(
+  input: AsyncIterable<Uint8Array>,
+): AsyncGenerator<Float32Array> {
+  for await (const item of decodeMsgpack<Uint8Array>(wrapIterable(input))) {
+    const aligned = alignBuffer(item);
+    yield new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
+  }
+}
+
+function alignBuffer(bytes: Uint8Array): Uint8Array {
+  if (bytes.byteOffset % 4 === 0) return bytes;
+  const aligned = new Uint8Array(bytes.length);
+  aligned.set(bytes);
+  return aligned;
+}
+
+function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
+  if (a.length === 0) return b;
+  const result = new Uint8Array(a.length + b.length);
+  result.set(a, 0);
+  result.set(b, a.length);
+  return result;
+}
+
+async function* wrapIterable<T>(input: AsyncIterable<T>): AsyncGenerator<T> {
+  yield* input;
+}