@effected/yaml 0.6.1 → 0.8.0

package/YamlToken.js

@@ -0,0 +1,139 @@
+import { lexAll } from "./internal/lexer.js";
+import { Result, Schema, Stream } from "effect";
+
+//#region src/YamlToken.ts
+/**
+* The 22 lexical token kinds produced by the YAML tokenizer.
+*
+* @public
+*/
+const YamlTokenKind = Schema.Literals([
+	"document-start",
+	"document-end",
+	"directive",
+	"tag",
+	"anchor",
+	"alias",
+	"scalar",
+	"block-map-start",
+	"block-map-key",
+	"block-map-value",
+	"block-seq-start",
+	"block-seq-entry",
+	"flow-map-start",
+	"flow-map-end",
+	"flow-seq-start",
+	"flow-seq-end",
+	"flow-separator",
+	"newline",
+	"whitespace",
+	"comment",
+	"byte-order-mark",
+	"error"
+]);
+/**
+* A single positioned YAML lexical token.
+*
+* - `kind` — the {@link (YamlTokenKind:type)}.
+* - `text` — the raw source slice the token covers; the position-fidelity
+*   invariant is `source.slice(offset, offset + length) === text`.
+* - `offset` / `length` — the token's span in UTF-16 code units.
+* - `line` / `character` — the zero-based position of the token's start,
+*   matching the `YamlDiagnostic` position vocabulary.
+*
+* @public
+*/
+var YamlToken = class extends Schema.Class("YamlToken")({
+	kind: YamlTokenKind,
+	text: Schema.String,
+	offset: Schema.Number,
+	length: Schema.Number,
+	line: Schema.Number,
+	character: Schema.Number
+}) {};
+/**
+* Promote the internal lexer tokens to the public shape.
+*
+* `line`/`character` are DERIVED from each token's offset against a
+* line-start index rather than copied from the internal token: the internal
+* `column` is a CST-parser vocabulary that carries the construct's INDENT on
+* the synthetic `block-map-start`/`block-seq-start` markers, not the token's
+* own position — the public surface promises the position. `text` is the RAW
+* source slice for the same reason: the internal `value` is the PROCESSED
+* form on quoted scalars (content without the quotes), while the public
+* contract is byte fidelity — the corpus conformance suite pins the tokens
+* tiling the source exactly.
+*/
+const promoteAll = (text, tokens) => {
+	const lineStarts = [0];
+	for (let i = 0; i < text.length; i++) if (text[i] === "\n") lineStarts.push(i + 1);
+	let line = 0;
+	return tokens.map((token) => {
+		if (token.offset < lineStarts[line]) line = 0;
+		while (line + 1 < lineStarts.length && lineStarts[line + 1] <= token.offset) line++;
+		return new YamlToken({
+			kind: token.kind,
+			text: text.slice(token.offset, token.offset + token.length),
+			offset: token.offset,
+			length: token.length,
+			line,
+			character: token.offset - lineStarts[line]
+		});
+	});
+};
+/**
+* Tokenizer statics. Not instantiable.
+*
+* @public
+*/
+var YamlTokens = class YamlTokens {
+	constructor() {}
+	/**
+	* Tokenize YAML text into the full positioned token array — the sync
+	* `Result` primitive (tokenizing is a pure batch transform; the
+	* {@link YamlTokens.stream} form is derived from this one, per the kit's
+	* sync-primitive policy).
+	*
+	* @remarks
+	* The failure channel is **reserved** (for future input-hardening guards)
+	* and never fires today: the lexer is total, and lexical errors surface as
+	* `"error"`-kind tokens **in the success array** so that linting can run
+	* on malformed input — the `parse-validity` lint rule exists precisely for
+	* documents that do not parse. Do not "fix" this method to fail on
+	* `"error"` tokens; that would make malformed documents unlintable.
+	*
+	* @example
+	* ```ts
+	* import { YamlTokens } from "@effected/yaml";
+	* import { Result } from "effect";
+	*
+	* const result = YamlTokens.tokenize("a: 1\n");
+	* if (Result.isSuccess(result)) {
+	*   result.success.map((t) => t.kind); // ["scalar", "block-map-start", ...]
+	* }
+	* ```
+	*/
+	static tokenize(text) {
+		return Result.succeed(promoteAll(text, lexAll(text)));
+	}
+	/**
+	* Tokenize YAML text as a lazy `Stream` of tokens — the derived form of
+	* {@link YamlTokens.tokenize} for genuinely incremental (SAX-style)
+	* consumers, parallel to `YamlVisitor.visit`.
+	*
+	* @remarks
+	* Derived from the sync primitive, so it shares its contract: lexical
+	* errors arrive as `"error"`-kind tokens in the stream, never as a stream
+	* failure. (The primitive's reserved failure channel would surface as a
+	* defect here; it never fires today.)
+	*/
+	static stream(text) {
+		return Stream.suspend(() => Result.match(YamlTokens.tokenize(text), {
+			onSuccess: (tokens) => Stream.fromIterable(tokens),
+			onFailure: (error) => Stream.die(error)
+		}));
+	}
+};
+
+//#endregion
+export { YamlToken, YamlTokenKind, YamlTokens };

package/YamlVisitor.js

@@ -79,9 +79,16 @@ function* walkDocument(doc, text) {
 	if (doc.comment !== void 0) yield YamlVisitorEvent.Comment({
 		path,
 		depth,
-		text: doc.comment
+		text: doc.comment,
+		placement: "leading"
 	});
 	if (doc.contents !== null) yield* walkNode(doc.contents, path, depth);
+	if (doc.commentAfter !== void 0) yield YamlVisitorEvent.Comment({
+		path,
+		depth,
+		text: doc.commentAfter,
+		placement: "trailing"
+	});
 	yield YamlVisitorEvent.DocumentEnd({
 		path,
 		depth
@@ -89,10 +96,17 @@ function* walkDocument(doc, text) {
 }
 function* walkNode(node, path, depth) {
 	if (node instanceof YamlScalar) {
+		if (node.commentBefore !== void 0) yield YamlVisitorEvent.Comment({
+			path,
+			depth,
+			text: node.commentBefore,
+			placement: "leading"
+		});
 		if (node.comment !== void 0) yield YamlVisitorEvent.Comment({
 			path,
 			depth,
-			text: node.comment
+			text: node.comment,
+			placement: "trailing"
 		});
 		yield YamlVisitorEvent.Scalar({
 			path,
@@ -108,10 +122,17 @@ function* walkNode(node, path, depth) {
 		name: node.name
 	});
 	else if (node instanceof YamlMap) {
+		if (node.commentBefore !== void 0) yield YamlVisitorEvent.Comment({
+			path,
+			depth,
+			text: node.commentBefore,
+			placement: "leading"
+		});
 		if (node.comment !== void 0) yield YamlVisitorEvent.Comment({
 			path,
 			depth,
-			text: node.comment
+			text: node.comment,
+			placement: "trailing"
 		});
 		yield YamlVisitorEvent.MapStart({
 			path,
@@ -126,10 +147,17 @@ function* walkNode(node, path, depth) {
 			depth
 		});
 	} else if (node instanceof YamlSeq) {
+		if (node.commentBefore !== void 0) yield YamlVisitorEvent.Comment({
+			path,
+			depth,
+			text: node.commentBefore,
+			placement: "leading"
+		});
 		if (node.comment !== void 0) yield YamlVisitorEvent.Comment({
 			path,
 			depth,
-			text: node.comment
+			text: node.comment,
+			placement: "trailing"
 		});
 		yield YamlVisitorEvent.SeqStart({
 			path,
@@ -153,10 +181,17 @@ function* walkPair(pair, parentPath, depth) {
 	const resolvedValue = pair.value instanceof YamlScalar ? pair.value.value : null;
 	const keySegment = typeof resolvedKey === "string" ? resolvedKey : typeof resolvedKey === "number" ? resolvedKey : String(resolvedKey);
 	const pairPath = [...parentPath, keySegment];
+	if (pair.commentBefore !== void 0) yield YamlVisitorEvent.Comment({
+		path: pairPath,
+		depth,
+		text: pair.commentBefore,
+		placement: "leading"
+	});
 	if (pair.comment !== void 0) yield YamlVisitorEvent.Comment({
 		path: pairPath,
 		depth,
-		text: pair.comment
+		text: pair.comment,
+		placement: "trailing"
 	});
 	yield YamlVisitorEvent.Pair({
 		path: pairPath,