@effected/toml 0.1.0

package/internal/scanner.js

@@ -0,0 +1,493 @@
+import { RawTomlError } from "./diagnostics.js";
+import { TomlLocalDate, TomlLocalDateTime, TomlLocalTime, TomlOffsetDateTime } from "../TomlDateTime.js";
+
+//#region src/internal/scanner.ts
+const NUL = 0;
+const TAB = 9;
+const LF = 10;
+const CR = 13;
+const SPACE = 32;
+const QUOTE = 34;
+const HASH = 35;
+const APOSTROPHE = 39;
+const COMMA = 44;
+const HYPHEN = 45;
+const BACKSLASH = 92;
+const RIGHT_BRACKET = 93;
+const RIGHT_BRACE = 125;
+const LOWER_U = 117;
+const UPPER_U = 85;
+const BOM = 65279;
+const raise = (code, message, offset, length) => {
+	throw new RawTomlError({
+		code,
+		message,
+		offset,
+		length
+	});
+};
+/** `U+XXXX` display form of a char code. */
+const codeLabel = (code) => `U+${code.toString(16).toUpperCase().padStart(4, "0")}`;
+/** TOML control characters (tab excluded): U+0000-U+0008, U+000A-U+001F, U+007F. */
+const isControlChar = (code) => code <= 8 || code >= 10 && code <= 31 || code === 127;
+const isDigit = (code) => code >= 48 && code <= 57;
+/** Whether `code` is a bare-key character: `[A-Za-z0-9_-]`. */
+const isBareKeyChar = (code) => code >= 97 && code <= 122 || code >= 65 && code <= 90 || isDigit(code) || code === 95 || code === HYPHEN;
+/** The number of code units to skip for a single leading U+FEFF BOM. */
+const skipBom = (source) => source.charCodeAt(0) === BOM ? 1 : 0;
+/**
+* Reject a document carrying U+FFFD REPLACEMENT CHARACTER. TOML 1.0.0 requires
+* a valid UTF-8 document, but this engine receives an already-decoded JS
+* string — after Node's lossy utf8 decode, U+FFFD is the only surviving
+* evidence of a malformed byte sequence. The grammar technically admits
+* U+FFFD inside strings and comments, so this trades away a pathological
+* legal character to honor the encoding rule the toml-test corpus pins
+* (invalid/encoding/bad-utf8-in-string.toml, bad-utf8-in-comment.toml,
+* bad-codepoint.toml and friends).
+*/
+const assertValidUnicode = (source) => {
+	const index = source.indexOf("�");
+	if (index !== -1) raise("InvalidUtf8", "invalid UTF-8: U+FFFD REPLACEMENT CHARACTER marks a malformed byte sequence in the decoded input", index, 1);
+};
+/** Skip spaces and tabs; returns the position of the first other character. */
+const scanWhitespace = (source, pos) => {
+	let i = pos;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code !== SPACE && code !== TAB) break;
+		i += 1;
+	}
+	return i;
+};
+/**
+* Consume one `\n` or `\r\n` newline. A lone `\r` throws BareCarriageReturn;
+* no newline at `pos` returns `pos` unchanged.
+*/
+const scanNewline = (source, pos) => {
+	const code = source.charCodeAt(pos);
+	if (code === LF) return pos + 1;
+	if (code === CR) {
+		if (source.charCodeAt(pos + 1) === LF) return pos + 2;
+		return raise("BareCarriageReturn", "carriage return not followed by a line feed", pos, 1);
+	}
+	return pos;
+};
+/**
+* Scan a comment starting at `#` through end of line or EOF. The value
+* excludes the `#`; control characters other than tab are rejected.
+*/
+const scanComment = (source, pos) => {
+	let i = pos + 1;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === LF || code === CR) break;
+		if (isControlChar(code)) return raise("ControlCharacterInComment", `control character ${codeLabel(code)} in comment`, i, 1);
+		i += 1;
+	}
+	return {
+		value: source.slice(pos + 1, i),
+		end: i
+	};
+};
+/** Scan a run of bare-key characters; the value may be empty. */
+const scanBareKey = (source, pos) => {
+	let i = pos;
+	while (i < source.length && isBareKeyChar(source.charCodeAt(i))) i += 1;
+	return {
+		value: source.slice(pos, i),
+		end: i
+	};
+};
+/** The decoded character for a simple escape code, or undefined. */
+const simpleEscape = (code) => {
+	switch (code) {
+		case 98: return "\b";
+		case 116: return "	";
+		case 110: return "\n";
+		case 102: return "\f";
+		case 114: return "\r";
+		case QUOTE: return "\"";
+		case BACKSLASH: return "\\";
+		default: return;
+	}
+};
+const HEX_DIGITS = /^[0-9A-Fa-f]+$/;
+/**
+* Decode a `\uXXXX` / `\UXXXXXXXX` escape at `backslash`. The code point must
+* be a Unicode scalar value: surrogates and values above U+10FFFF are errors.
+*/
+const decodeUnicodeEscape = (source, backslash, width) => {
+	const start = backslash + 2;
+	const hex = source.slice(start, start + width);
+	if (hex.length < width || !HEX_DIGITS.test(hex)) return raise("InvalidUnicodeEscape", `\\${width === 4 ? "u" : "U"} escape requires ${width} hexadecimal digits`, backslash, width + 2);
+	const codePoint = Number.parseInt(hex, 16);
+	if (codePoint >= 55296 && codePoint <= 57343 || codePoint > 1114111) return raise("InvalidUnicodeEscape", `${codeLabel(codePoint)} is not a Unicode scalar value`, backslash, width + 2);
+	return {
+		codePoint,
+		end: start + width
+	};
+};
+/** Scan a single-line basic string starting at the opening `"`. */
+const scanBasicString = (source, pos) => {
+	let out = "";
+	let chunkStart = pos + 1;
+	let i = pos + 1;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === QUOTE) return {
+			value: out + source.slice(chunkStart, i),
+			end: i + 1
+		};
+		if (code === BACKSLASH) {
+			if (i + 1 >= source.length) break;
+			out += source.slice(chunkStart, i);
+			const next = source.charCodeAt(i + 1);
+			const mapped = simpleEscape(next);
+			if (mapped !== void 0) {
+				out += mapped;
+				i += 2;
+			} else if (next === LOWER_U || next === UPPER_U) {
+				const decoded = decodeUnicodeEscape(source, i, next === LOWER_U ? 4 : 8);
+				out += String.fromCodePoint(decoded.codePoint);
+				i = decoded.end;
+			} else return raise("InvalidEscape", `invalid escape sequence \\${source[i + 1] ?? ""}`, i, 2);
+			chunkStart = i;
+			continue;
+		}
+		if (code === LF || code === CR && source.charCodeAt(i + 1) === LF) return raise("UnterminatedString", "basic string not closed before end of line", pos, i - pos);
+		if (code === CR) return raise("BareCarriageReturn", "carriage return not followed by a line feed", i, 1);
+		if (isControlChar(code)) return raise("ControlCharacterInString", `control character ${codeLabel(code)} in basic string`, i, 1);
+		i += 1;
+	}
+	return raise("UnterminatedString", "unterminated basic string", pos, source.length - pos);
+};
+/** Scan a single-line literal string starting at the opening `'`. */
+const scanLiteralString = (source, pos) => {
+	let i = pos + 1;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === APOSTROPHE) return {
+			value: source.slice(pos + 1, i),
+			end: i + 1
+		};
+		if (code === LF || code === CR && source.charCodeAt(i + 1) === LF) return raise("UnterminatedString", "literal string not closed before end of line", pos, i - pos);
+		if (code === CR) return raise("BareCarriageReturn", "carriage return not followed by a line feed", i, 1);
+		if (isControlChar(code)) return raise("ControlCharacterInString", `control character ${codeLabel(code)} in literal string`, i, 1);
+		i += 1;
+	}
+	return raise("UnterminatedString", "unterminated literal string", pos, source.length - pos);
+};
+/** Skip a newline immediately after a multiline opening delimiter. */
+const skipLeadingNewline = (source, pos) => {
+	if (source.charCodeAt(pos) === LF) return pos + 1;
+	if (source.charCodeAt(pos) === CR && source.charCodeAt(pos + 1) === LF) return pos + 2;
+	return pos;
+};
+/** Skip whitespace and newlines after a line-ending backslash's newline. */
+const skipLineEndingTrim = (source, pos) => {
+	let i = pos;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === SPACE || code === TAB || code === LF) {
+			i += 1;
+			continue;
+		}
+		if (code === CR) {
+			if (source.charCodeAt(i + 1) === LF) {
+				i += 2;
+				continue;
+			}
+			return raise("BareCarriageReturn", "carriage return not followed by a line feed", i, 1);
+		}
+		break;
+	}
+	return i;
+};
+/** Scan a multiline basic string starting at the opening `"""`. */
+const scanMultilineBasicString = (source, pos) => {
+	let i = skipLeadingNewline(source, pos + 3);
+	let out = "";
+	let chunkStart = i;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === QUOTE) {
+			let runEnd = i;
+			while (runEnd < source.length && source.charCodeAt(runEnd) === QUOTE) runEnd += 1;
+			const run = runEnd - i;
+			if (run < 3) {
+				i = runEnd;
+				continue;
+			}
+			const contentQuotes = Math.min(run - 3, 2);
+			return {
+				value: out + source.slice(chunkStart, i + contentQuotes),
+				end: i + contentQuotes + 3
+			};
+		}
+		if (code === BACKSLASH) {
+			out += source.slice(chunkStart, i);
+			const next = source.charCodeAt(i + 1);
+			const mapped = simpleEscape(next);
+			if (mapped !== void 0) {
+				out += mapped;
+				i += 2;
+				chunkStart = i;
+				continue;
+			}
+			if (next === LOWER_U || next === UPPER_U) {
+				const decoded = decodeUnicodeEscape(source, i, next === LOWER_U ? 4 : 8);
+				out += String.fromCodePoint(decoded.codePoint);
+				i = decoded.end;
+				chunkStart = i;
+				continue;
+			}
+			let probe = i + 1;
+			while (probe < source.length) {
+				const probeCode = source.charCodeAt(probe);
+				if (probeCode !== SPACE && probeCode !== TAB) break;
+				probe += 1;
+			}
+			if (probe >= source.length) break;
+			const probeCode = source.charCodeAt(probe);
+			if (probeCode === LF || probeCode === CR && source.charCodeAt(probe + 1) === LF) {
+				i = skipLineEndingTrim(source, probe);
+				chunkStart = i;
+				continue;
+			}
+			return raise("InvalidEscape", `invalid escape sequence \\${source[i + 1] ?? ""}`, i, probe - i + 1);
+		}
+		if (code === LF) {
+			i += 1;
+			continue;
+		}
+		if (code === CR) {
+			if (source.charCodeAt(i + 1) === LF) {
+				i += 2;
+				continue;
+			}
+			return raise("BareCarriageReturn", "carriage return not followed by a line feed", i, 1);
+		}
+		if (isControlChar(code)) return raise("ControlCharacterInString", `control character ${codeLabel(code)} in multiline basic string`, i, 1);
+		i += 1;
+	}
+	return raise("UnterminatedString", "unterminated multiline basic string", pos, source.length - pos);
+};
+/** Scan a multiline literal string starting at the opening `'''`. */
+const scanMultilineLiteralString = (source, pos) => {
+	const contentStart = skipLeadingNewline(source, pos + 3);
+	let i = contentStart;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === APOSTROPHE) {
+			let runEnd = i;
+			while (runEnd < source.length && source.charCodeAt(runEnd) === APOSTROPHE) runEnd += 1;
+			const run = runEnd - i;
+			if (run < 3) {
+				i = runEnd;
+				continue;
+			}
+			const contentQuotes = Math.min(run - 3, 2);
+			return {
+				value: source.slice(contentStart, i + contentQuotes),
+				end: i + contentQuotes + 3
+			};
+		}
+		if (code === LF) {
+			i += 1;
+			continue;
+		}
+		if (code === CR) {
+			if (source.charCodeAt(i + 1) === LF) {
+				i += 2;
+				continue;
+			}
+			return raise("BareCarriageReturn", "carriage return not followed by a line feed", i, 1);
+		}
+		if (isControlChar(code)) return raise("ControlCharacterInString", `control character ${codeLabel(code)} in multiline literal string`, i, 1);
+		i += 1;
+	}
+	return raise("UnterminatedString", "unterminated multiline literal string", pos, source.length - pos);
+};
+/** Scan one raw token span; NUL anywhere is a lex error. */
+const scanTokenSpan = (source, pos) => {
+	let i = pos;
+	while (i < source.length) {
+		const code = source.charCodeAt(i);
+		if (code === NUL) return raise("InvalidCharacter", "NUL character in document", i, 1);
+		if (code === SPACE || code === TAB || code === LF || code === CR || code === COMMA || code === RIGHT_BRACKET || code === RIGHT_BRACE || code === HASH) break;
+		i += 1;
+	}
+	return i;
+};
+const OFFSET_DATE_TIME = /^([0-9]{4})-([0-9]{2})-([0-9]{2})[Tt ]([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]+))?([Zz]|[+-][0-9]{2}:[0-9]{2})$/;
+const LOCAL_DATE_TIME = /^([0-9]{4})-([0-9]{2})-([0-9]{2})[Tt ]([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]+))?$/;
+const LOCAL_DATE = /^([0-9]{4})-([0-9]{2})-([0-9]{2})$/;
+const LOCAL_TIME = /^([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]+))?$/;
+/**
+* Scan a non-string scalar value token. Stops at whitespace, newlines, `,`,
+* `]`, `}` and `#` — with the G5 extension: a token that scanned as a full
+* date followed by a single space and a digit continues through the time
+* part, so `1979-05-27 07:32:00Z` is one token.
+*/
+const scanValueToken = (source, pos) => {
+	let end = scanTokenSpan(source, pos);
+	if (LOCAL_DATE.test(source.slice(pos, end)) && source.charCodeAt(end) === SPACE && isDigit(source.charCodeAt(end + 1))) end = scanTokenSpan(source, end + 1);
+	return {
+		value: source.slice(pos, end),
+		end
+	};
+};
+const INTEGER_DEC = /^[+-]?(?:0|[1-9](?:_?[0-9])*)$/;
+const INTEGER_HEX = /^0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*$/;
+const INTEGER_OCT = /^0o[0-7](?:_?[0-7])*$/;
+const INTEGER_BIN = /^0b[01](?:_?[01])*$/;
+const FLOAT = /^[+-]?(?:0|[1-9](?:_?[0-9])*)(?:\.[0-9](?:_?[0-9])*(?:[eE][+-]?[0-9](?:_?[0-9])*)?|[eE][+-]?[0-9](?:_?[0-9])*)$/;
+const FLOAT_SPECIAL = /^[+-]?(?:inf|nan)$/;
+/** A token that starts number-shaped fails as InvalidNumber, not InvalidValue. */
+const NUMERIC_LOOKING = /^[+-]?[0-9.]/;
+const INT64_MIN = -(2n ** 63n);
+const INT64_MAX = 2n ** 63n - 1n;
+const MAX_SAFE_BIG = 2n ** 53n - 1n;
+/** Range-check against int64 and narrow to number when within 2^53-1. */
+const narrowInteger = (big, token, offset) => {
+	if (big < INT64_MIN || big > INT64_MAX) return raise("IntegerOutOfRange", `${token} does not fit in a signed 64-bit integer`, offset, token.length);
+	return big >= -MAX_SAFE_BIG && big <= MAX_SAFE_BIG ? Number(big) : big;
+};
+const decodeDecimalInteger = (token, offset) => {
+	let digits = token.replace(/_/g, "");
+	if (digits.startsWith("+")) digits = digits.slice(1);
+	return narrowInteger(BigInt(digits), token, offset);
+};
+const decodePrefixedInteger = (token, offset) => narrowInteger(BigInt(token.replace(/_/g, "")), token, offset);
+const MONTH_LENGTHS = [
+	31,
+	28,
+	31,
+	30,
+	31,
+	30,
+	31,
+	31,
+	30,
+	31,
+	30,
+	31
+];
+const isLeapYear = (year) => year % 4 === 0 && year % 100 !== 0 || year % 400 === 0;
+const validateDate = (year, month, day, offset, length) => {
+	if (month < 1 || month > 12) raise("InvalidDateTime", `month ${month} is out of range 1-12`, offset, length);
+	const max = month === 2 && isLeapYear(year) ? 29 : MONTH_LENGTHS[month - 1] ?? 0;
+	if (day < 1 || day > max) raise("InvalidDateTime", `day ${day} does not exist in month ${month} of ${year}`, offset, length);
+};
+const validateTime = (hour, minute, second, offset, length) => {
+	if (hour > 23) raise("InvalidDateTime", `hour ${hour} is out of range 0-23`, offset, length);
+	if (minute > 59) raise("InvalidDateTime", `minute ${minute} is out of range 0-59`, offset, length);
+	if (second > 60) raise("InvalidDateTime", `second ${second} is out of range 0-60`, offset, length);
+};
+/** Right-pad or truncate fractional-second digits to exactly nine → nanoseconds. */
+const decodeNanosecond = (fraction) => fraction === void 0 ? 0 : Number(`${fraction}000000000`.slice(0, 9));
+/** Decode `Z` / `z` / `[+-]hh:mm` to signed minutes, validating hh and mm. */
+const decodeOffsetMinutes = (text, offset, length) => {
+	if (text === "Z" || text === "z") return 0;
+	const hh = Number(text.slice(1, 3));
+	const mm = Number(text.slice(4, 6));
+	if (hh > 23) raise("InvalidDateTime", `offset hour ${hh} is out of range 0-23`, offset, length);
+	if (mm > 59) raise("InvalidDateTime", `offset minute ${mm} is out of range 0-59`, offset, length);
+	const total = hh * 60 + mm;
+	if (total === 0) return 0;
+	return text.charCodeAt(0) === HYPHEN ? -total : total;
+};
+/**
+* Classify a scanned value token per G4-G6: booleans, the four datetime
+* shapes (validated against the Gregorian calendar and clock ranges before
+* construction), integers across four radixes with int64 range checking and
+* number/bigint narrowing, and floats including the special spellings.
+*/
+const classifyValueToken = (token, offset) => {
+	if (token === "true") return true;
+	if (token === "false") return false;
+	const length = Math.max(token.length, 1);
+	let match = OFFSET_DATE_TIME.exec(token);
+	if (match !== null) {
+		const [, y = "", mo = "", d = "", h = "", mi = "", s = "", fraction, offsetText = ""] = match;
+		const year = Number(y);
+		const month = Number(mo);
+		const day = Number(d);
+		const hour = Number(h);
+		const minute = Number(mi);
+		const second = Number(s);
+		validateDate(year, month, day, offset, length);
+		validateTime(hour, minute, second, offset, length);
+		const offsetMinutes = decodeOffsetMinutes(offsetText, offset, length);
+		return new TomlOffsetDateTime({
+			year,
+			month,
+			day,
+			hour,
+			minute,
+			second,
+			nanosecond: decodeNanosecond(fraction),
+			offsetMinutes
+		});
+	}
+	match = LOCAL_DATE_TIME.exec(token);
+	if (match !== null) {
+		const [, y = "", mo = "", d = "", h = "", mi = "", s = "", fraction] = match;
+		const year = Number(y);
+		const month = Number(mo);
+		const day = Number(d);
+		const hour = Number(h);
+		const minute = Number(mi);
+		const second = Number(s);
+		validateDate(year, month, day, offset, length);
+		validateTime(hour, minute, second, offset, length);
+		return new TomlLocalDateTime({
+			year,
+			month,
+			day,
+			hour,
+			minute,
+			second,
+			nanosecond: decodeNanosecond(fraction)
+		});
+	}
+	match = LOCAL_DATE.exec(token);
+	if (match !== null) {
+		const [, y = "", mo = "", d = ""] = match;
+		const year = Number(y);
+		const month = Number(mo);
+		const day = Number(d);
+		validateDate(year, month, day, offset, length);
+		return new TomlLocalDate({
+			year,
+			month,
+			day
+		});
+	}
+	match = LOCAL_TIME.exec(token);
+	if (match !== null) {
+		const [, h = "", mi = "", s = "", fraction] = match;
+		const hour = Number(h);
+		const minute = Number(mi);
+		const second = Number(s);
+		validateTime(hour, minute, second, offset, length);
+		return new TomlLocalTime({
+			hour,
+			minute,
+			second,
+			nanosecond: decodeNanosecond(fraction)
+		});
+	}
+	if (INTEGER_DEC.test(token)) return decodeDecimalInteger(token, offset);
+	if (INTEGER_HEX.test(token) || INTEGER_OCT.test(token) || INTEGER_BIN.test(token)) return decodePrefixedInteger(token, offset);
+	if (FLOAT.test(token)) return Number(token.replace(/_/g, ""));
+	if (FLOAT_SPECIAL.test(token)) {
+		if (token.endsWith("nan")) return NaN;
+		return token.charCodeAt(0) === HYPHEN ? Number.NEGATIVE_INFINITY : Number.POSITIVE_INFINITY;
+	}
+	if (NUMERIC_LOOKING.test(token)) return raise("InvalidNumber", `${token} is not a valid TOML number or date-time`, offset, length);
+	return raise("InvalidValue", `${token} is not a valid TOML value`, offset, length);
+};
+
+//#endregion
+export { assertValidUnicode, classifyValueToken, isBareKeyChar, scanBareKey, scanBasicString, scanComment, scanLiteralString, scanMultilineBasicString, scanMultilineLiteralString, scanNewline, scanValueToken, scanWhitespace, skipBom };