@effected/jsonl 0.8.0 → 0.10.0

package/internal/tail.js

@@ -1,4 +1,4 @@
-import { ByteSize, Effect, Option } from "effect";
+import { ByteSize, Effect, Option, Stream } from "effect";
 
 //#region src/internal/tail.ts
 /** UTF-8 BOM, as bytes. `U+FEFF` encodes to these three. */
@@ -10,134 +10,217 @@ const BOM = [
 /** Byte value of `\n`. Cannot occur inside a UTF-8 multi-byte sequence. */
 const LF = 10;
 /**
-* The default tail window.
-*
-* Large enough that a snapshot journal's last line is almost always inside it
-* on the first read, small enough that reading it is cheap against a journal of
-* any age. When it misses, {@link readTail} widens rather than failing.
+* The first tail window. Large enough that a snapshot journal's last line is
+* almost always inside it, small enough to be cheap against a journal of any
+* age.
 *
 * @internal
 */
 const DEFAULT_WINDOW = 8192;
-/** Does the buffer begin with a UTF-8 BOM? */
-const hasBom = (bytes) => bytes.length >= 3 && bytes[0] === BOM[0] && bytes[1] === BOM[1] && bytes[2] === BOM[2];
 /**
-* Probe the first three bytes of a file for a BOM.
+* The largest window a single tail read allocates.
 *
-* This is a property of the FILE, so it is read once from the start rather than
-* inferred from whatever window happens to be in hand. Inferring it from window
-* position was a real defect: a BOM'd journal larger than the window never has
-* a window at offset 0, so every offset it emitted was physical — silently off
-* by three against a file the package itself had described as post-BOM.
+* A regression fence: the historical read once sized a "tail" window to its
+* whole region. A caller needing more than this pages for it.
 *
 * @internal
 */
-const probeBomBytes = (fs, path) => Effect.gen(function* () {
-	const head = yield* (yield* fs.open(path, { flag: "r" })).readAlloc(BOM.length);
-	const bytes = Option.getOrElse(head, () => /* @__PURE__ */ new Uint8Array(0));
-	return hasBom(bytes) ? BOM.length : 0;
-}).pipe(Effect.scoped);
+const MAX_WINDOW = 1048576;
 /**
-* Read the last `window` bytes of a journal, decoded from a line boundary.
-*
-* Three disciplines, each load-bearing:
+* The forward page size of {@link readLinePages}.
 *
-* 1. **Start at a line boundary.** Unless the window reaches offset 0, the
-*    leading partial line is discarded — everything up to and including the
-*    first `\n`. This is also what makes the decode safe across a chunk
-*    boundary: `\n` is `0x0A` and **cannot** appear inside a UTF-8 multi-byte
-*    sequence, so a window that starts just past one starts on a character
-*    boundary too. No separate mid-character guard is needed.
-* 2. **Strip exactly one leading BOM, at the byte level**, and only when the
-*    window genuinely starts at file offset 0. Reads here are offset-based, so
-*    an implicit strip (as `readFileString` performs) would desynchronize every
-*    offset the package emits; doing it explicitly keeps the convention stated.
-*    A `U+FEFF` anywhere else is content and is left alone.
-* 3. **Never read the whole file** unless the file is smaller than the window.
+* @internal
+*/
+const PAGE_SIZE = 65536;
+const EMPTY = /* @__PURE__ */ new Uint8Array(0);
+const decoder = new TextDecoder();
+/** Join byte chunks with one allocation. */
+const join = (chunks) => {
+	if (chunks.length === 1) return chunks[0];
+	let length = 0;
+	for (const chunk of chunks) length += chunk.length;
+	const joined = new Uint8Array(length);
+	let at = 0;
+	for (const chunk of chunks) {
+		joined.set(chunk, at);
+		at += chunk.length;
+	}
+	return joined;
+};
+/** Read `[from, to)` (physical) from an open handle. */
+const readAt = (file, from, to) => Effect.gen(function* () {
+	yield* file.seek(BigInt(from), "start");
+	return Option.getOrElse(yield* file.readAlloc(to - from), () => EMPTY);
+});
+/**
+* Probe the first three bytes of a file for a BOM.
 *
-* **`window` is not clamped**, deliberately. `Journal`'s historical read sizes
-* its window to the region it has to return, so a clamp here would silently
-* truncate that read rather than bound it. The clamp belongs with the paged
-* rewrite of that read — one that emits per window and can therefore honour a
-* maximum — and is carried on spencerbeggs/effected#233, not added underneath
-* it.
+* A property of the FILE, read once from the start — never inferred from a
+* window's position, which for a journal larger than the window never reaches
+* offset 0.
 *
 * @internal
 */
-const readTail = (fs, path, window, bomBytes) => Effect.gen(function* () {
-	const info = yield* fs.stat(path);
-	const physicalSize = ByteSize.toNumberUnsafe(info.size);
-	const logicalSize = physicalSize - bomBytes;
-	const from = Math.max(bomBytes, physicalSize - window);
+const probeBomBytes = (fs, path) => Effect.gen(function* () {
 	const file = yield* fs.open(path, { flag: "r" });
-	yield* file.seek(BigInt(from), "start");
-	const read = yield* file.readAlloc(physicalSize - from);
-	const bytes = Option.getOrElse(read, () => /* @__PURE__ */ new Uint8Array(0));
-	const windowAtFileStart = from === bomBytes;
+	const head = yield* readAt(file, 0, BOM.length);
+	return head.length >= 3 && head[0] === BOM[0] && head[1] === BOM[1] && head[2] === BOM[2] ? BOM.length : 0;
+}).pipe(Effect.scoped);
+/**
+* Read the window of at most {@link MAX_WINDOW} bytes ending at physical `end`,
+* decoded from a line boundary.
+*
+* Unless the window reaches the start of the content, its leading partial line
+* is discarded — through the first `\n`, which cannot occur inside a UTF-8
+* multi-byte sequence, so the decode starts on a character boundary too. A BOM
+* is excluded by starting the read past it, never by an implicit strip.
+*/
+const readWindow = (file, end, window, bomBytes) => Effect.gen(function* () {
+	const from = Math.max(bomBytes, end - Math.min(window, MAX_WINDOW));
+	const bytes = yield* readAt(file, from, end);
+	const atFileStart = from === bomBytes;
 	let cursor = 0;
-	if (!windowAtFileStart) {
+	if (!atFileStart) {
 		const newline = bytes.indexOf(LF);
 		cursor = newline === -1 ? bytes.length : newline + 1;
 	}
 	return {
-		text: new TextDecoder().decode(bytes.subarray(cursor)),
+		text: decoder.decode(bytes.subarray(cursor)),
 		start: from + cursor - bomBytes,
-		atFileStart: windowAtFileStart,
-		size: logicalSize
+		atFileStart
 	};
-}).pipe(Effect.scoped);
+});
+/**
+* Read the one line that ends at physical `end`, however long it is, in
+* clamped steps backward to the newline before it.
+*
+* The escape hatch for a window with no line boundary inside it: the line is
+* longer than the window, and the clamp forbids asking for a bigger one.
+*/
+const readLineEnding = (file, end, bomBytes) => Effect.gen(function* () {
+	const chunks = [];
+	let position = end;
+	for (;;) {
+		const from = Math.max(bomBytes, position - MAX_WINDOW);
+		const bytes = yield* readAt(file, from, position);
+		const searchFrom = position === end ? bytes.length - 2 : bytes.length - 1;
+		const newline = searchFrom < 0 ? -1 : bytes.lastIndexOf(LF, searchFrom);
+		if (newline !== -1) {
+			chunks.unshift(bytes.subarray(newline + 1));
+			return {
+				text: decoder.decode(join(chunks)),
+				start: from + newline + 1 - bomBytes,
+				atFileStart: false
+			};
+		}
+		chunks.unshift(bytes);
+		if (from === bomBytes) return {
+			text: decoder.decode(join(chunks)),
+			start: 0,
+			atFileStart: true
+		};
+		position = from;
+	}
+});
 /**
-* Read widening windows until `decode` finds something, or the whole file has
-* been seen.
+* Search backward from the end of the journal, one window at a time, until
+* `decode` finds something or the whole file has been seen.
 *
-* The widening is what makes the bounded read *correct* rather than merely
-* cheap: a journal whose last line is longer than the initial window would
-* otherwise report "no valid envelope" for a perfectly healthy file.
+* Each window covers only bytes no earlier one did — it ends at the line
+* boundary where the previous one began — and the windows grow from
+* `initialWindow` by fours up to {@link MAX_WINDOW}. `decode` sees each window
+* on its own: "the last match in this window" is the last match in the file
+* only because every later window has already answered none.
 *
 * @internal
 */
 const readTailUntil = (fs, path, bomBytes, decode, initialWindow = DEFAULT_WINDOW) => Effect.gen(function* () {
+	const file = yield* fs.open(path, { flag: "r" });
+	let end = ByteSize.toNumberUnsafe((yield* file.stat).size);
 	let window = initialWindow;
 	for (;;) {
-		const tail = yield* readTail(fs, path, window, bomBytes);
+		let tail = yield* readWindow(file, end, window, bomBytes);
+		if (tail.text === "" && !tail.atFileStart && tail.start + bomBytes === end) tail = yield* readLineEnding(file, end, bomBytes);
 		const found = decode(tail);
-		if (Option.isSome(found)) return found;
-		if (tail.atFileStart) return Option.none();
-		window *= 4;
+		if (Option.isSome(found) || tail.atFileStart) return found;
+		end = tail.start + bomBytes;
+		window = Math.min(window * 4, MAX_WINDOW);
 	}
-});
+}).pipe(Effect.scoped);
 /**
-* Read a byte range and decode it as text, safely across chunk boundaries.
+* Read the logical region `[from, end)` forward, in pages, as complete lines.
 *
-* The byte→string seam lives here, in the service layer, because `Line.split`
-* is string-in by design and the pure core must never learn about buffers.
+* Each pull reads at most `pageSize` bytes, emits every line that page
+* completes, and carries the trailing unterminated fragment into the next pull
+* — so a consumer that stops early never pays for the rest, and memory is a
+* page plus the longest line. Pages are cut at `\n`, so each decodes on its
+* own: a character a raw read split in two is still whole in the carry by the
+* time its line is emitted.
 *
-* `TextDecoder` is used in **streaming mode** (`{ stream: true }`) so a
-* multi-byte character split across two reads is reassembled rather than
-* mangled. A naive per-chunk `decode` corrupts any such character — a bug that
-* appears only with non-ASCII payloads at specific sizes, which is exactly the
-* kind that reaches production. The final `decode()` with no argument flushes
-* any trailing partial sequence.
+* `from` is a line cursor: at a line boundary it is exactly where reading
+* starts; pointing INTO a line, the straddled line is skipped whole. That is why
+* reading starts one byte early — a `from` at a line start hands the skip rule
+* the previous line's terminator and consumes exactly that byte. Whatever
+* remains at `end` without a terminator is emitted last.
+*
+* `end` is the caller's bound, sampled once: bytes appended past it are never
+* read. A file that shrinks underneath ends the stream at what was readable.
 *
 * @internal
 */
-const readRangeText = (fs, path, from, length) => Effect.gen(function* () {
-	if (length <= 0) return "";
+const readLinePages = (fs, path, from, end, bomBytes, pageSize = PAGE_SIZE) => Stream.unwrap(Effect.gen(function* () {
+	if (from >= end) return Stream.empty;
 	const file = yield* fs.open(path, { flag: "r" });
-	yield* file.seek(BigInt(from), "start");
-	const decoder = new TextDecoder();
-	let text = "";
-	let remaining = length;
-	while (remaining > 0) {
-		const chunk = yield* file.readAlloc(Math.min(remaining, CHUNK));
-		if (Option.isNone(chunk) || chunk.value.length === 0) break;
-		text += decoder.decode(chunk.value, { stream: true });
-		remaining -= chunk.value.length;
-	}
-	return text + decoder.decode();
-}).pipe(Effect.scoped);
-/** Read granularity for incremental tail reads. */
-const CHUNK = 65536;
+	const stop = end + bomBytes;
+	const flush = (state) => state.skipping || state.carry.length === 0 ? [] : [{
+		text: decoder.decode(join(state.carry)),
+		start: state.carryStart
+	}];
+	return Stream.paginate({
+		position: from === 0 ? bomBytes : from + bomBytes - 1,
+		carry: [],
+		carryStart: from,
+		skipping: from > 0
+	}, (state) => Effect.gen(function* () {
+		const bytes = state.position >= stop ? EMPTY : yield* readAt(file, state.position, Math.min(state.position + pageSize, stop));
+		if (bytes.length === 0) return [flush(state), Option.none()];
+		const position = state.position + bytes.length;
+		let fresh = bytes;
+		let carryStart = state.carryStart;
+		if (state.skipping) {
+			const newline = bytes.indexOf(LF);
+			if (newline === -1) return [[], Option.some({
+				...state,
+				position
+			})];
+			fresh = bytes.subarray(newline + 1);
+			carryStart = state.position + newline + 1 - bomBytes;
+		}
+		const last = fresh.lastIndexOf(LF);
+		if (last === -1) {
+			const carry = fresh.length === 0 ? state.carry : [...state.carry, fresh];
+			return [[], Option.some({
+				position,
+				carry,
+				carryStart,
+				skipping: false
+			})];
+		}
+		const page = {
+			text: decoder.decode(join([...state.carry, fresh.subarray(0, last + 1)])),
+			start: carryStart
+		};
+		let carryLength = 0;
+		for (const chunk of state.carry) carryLength += chunk.length;
+		const rest = fresh.slice(last + 1);
+		return [[page], Option.some({
+			position,
+			carry: rest.length === 0 ? [] : [rest],
+			carryStart: carryStart + carryLength + last + 1,
+			skipping: false
+		})];
+	}));
+}));
 
 //#endregion
-export { DEFAULT_WINDOW, probeBomBytes, readRangeText, readTail, readTailUntil };
+export { DEFAULT_WINDOW, MAX_WINDOW, PAGE_SIZE, probeBomBytes, readLinePages, readTailUntil };

package/internal/utf8.js

@@ -1,14 +1,5 @@
 //#region src/internal/utf8.ts
 /**
-* UTF-8 byte accounting.
-*
-* A zero-dependency leaf so that both the line splitter and the error taxonomy
-* can measure bytes without either importing the other — the same cycle
-* firewall the other pure-tier packages use.
-*
-* @internal
-*/
-/**
 * The UTF-8 byte length of a JavaScript string.
 *
 * Journal offsets are **bytes**, because that is what `FileSystem.stream`'s

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@effected/jsonl",
-	"version": "0.8.0",
+	"version": "0.10.0",
 	"private": false,
 	"description": "Append-only, schema-validated JSONL journals as a definable Effect service.",
 	"keywords": [
@@ -37,7 +37,7 @@
 		"./package.json": "./package.json"
 	},
 	"peerDependencies": {
-		"effect": "4.0.0-rc.118"
+		"effect": "^4.0.0"
 	},
 	"engines": {
 		"node": ">=24.11.0"