@effected/markdown 0.12.1 → 0.14.0

package/Markdown.js

@@ -295,6 +295,13 @@ var Markdown = class Markdown {
 	* rows describe a synthesized node, which is the case a test asserting on
 	* generated markdown actually has.
 	*
+	* **Text escaping is canonical unless a `Text` node opts out.** A `Text`
+	* carrying `escapeStyle: "literal"` is emitted verbatim apart from the
+	* escapes that protect block structure (a table cell's `|`, a newline in a
+	* single-line container, line-start block openers); its exact set is
+	* documented on {@link Text}. The opt-out is additive: a tree that never
+	* sets it serializes byte-identically to the canonical form.
+	*
 	* **Representability wins over the table.** The canonical form never emits
 	* text that would re-parse as something else, so a row yields where the two
 	* conflict. The case that reaches a consumer is the indented code block: an
@@ -364,7 +371,9 @@ var Markdown = class Markdown {
 	* admits spec mdast and strips everything outside it, so a `fenceChar`,
 	* `headingStyle`, `markerChar` or `delimiter` placed on a plain mdast tree
 	* before admission is silently dropped and the canonical default applies.
-	* Set them on the nodes that come back.
+	* Set them on the nodes that come back. The exception is `Text`'s
+	* `escapeStyle`, an emitter instruction rather than a source spelling,
+	* which the boundary admits.
 	*
 	* @param root - The document tree to serialize.
 	* @returns An `Effect` that succeeds with markdown source, or fails with

package/MarkdownNode.js

@@ -145,11 +145,60 @@ const TableAlign = Schema.Literals([
 * Text — a run of literal characters, with entity references and backslash
 * escapes already resolved into `value`.
 *
+* @remarks
+* `escapeStyle` is an opt-in emitter extra, the one fidelity field the
+* parser never sets: absent (or `"canonical"`), `Markdown.stringify` escapes
+* the value canonically so it re-parses to the same text. `"literal"` is the
+* caller vouching that `value` is already safe markdown source, so the
+* emitter writes it verbatim and applies no escaping aimed at inline syntax:
+* `~0.2.1`, `^1.0.0`, `some_pkg` and `@scope/pkg` come out as written, as do
+* `*`, `_`, `[`, `]`, `~`, `&`, `\`, the backtick and autolink-shaped
+* text.
+*
+* Escaping that defends the containing BLOCK'S structure still applies,
+* because its absence would corrupt the document around the text rather
+* than add formatting inside it:
+*
+* - in a table cell, every `|` not already backslash-escaped is written
+*   `\|` (a bare pipe splits the cell);
+*
+* - in any container, a value-final `\` is doubled when more content
+*   follows it (it would otherwise pair with that content's first
+*   character, escaping an inline-code backtick or freeing a cell's `\|`);
+*
+* - in a table cell or heading, a newline becomes a space (neither can
+*   hold a line break);
+*
+* - in a heading, a trailing `#` run that would read as the ATX closing
+*   sequence is escaped;
+*
+* - elsewhere, a newline that would form a blank line (ending the
+*   paragraph) becomes `
`, and at every line start leading whitespace
+*   becomes a character reference, and a line start that can open a block
+*   is escaped: a `-`, `+` or `*` bullet or an ordered-list marker followed
+*   by a space, tab or line end; one to six `#` followed by the same; `>`;
+*   three or more backticks or tildes; a thematic-break or setext run
+*   (`***`, `___`, `==`); a `---` or `+++` frontmatter fence; a line of
+*   only `-`, `:`, `|` and whitespace; and, conservatively, any `<`, `[`,
+*   `|` or `:`. Text that cannot open a block there, such as `~0.2.1`,
+*   `1.0.0`, `-x` or `#x`, is written as is;
+*
+* - in a tree carrying MDX nodes, `{` and `<` stay escaped, since MDX reads
+*   a stray one as a syntax error rather than as text.
+*
+* **A literal text whose value parses as markdown does not round-trip to the
+* same value**: `*a*` emits verbatim and re-parses as emphasis. Keeping the
+* value free of inline syntax is the caller's promise, not something the
+* emitter checks. `Mdast.fromMdast` admits this field on a `text` node (it is
+* the one emitter instruction a plain tree can carry in), and `Mdast.toMdast`
+* projects it back out when present.
+*
 * @public
 */
 var Text = class extends Schema.Class("Text")({
 	type: Schema.tag("text"),
 	value: Schema.String,
+	escapeStyle: Schema.optionalKey(Schema.Literals(["canonical", "literal"])),
 	position: NodePosition
 }) {};
 /**

package/Mdast.js

@@ -110,7 +110,12 @@ const projectNode = (node) => {
 			children: projectChildren(node.children),
 			position
 		};
-		case "text":
+		case "text": return {
+			type: "text",
+			value: node.value,
+			...node.escapeStyle === void 0 ? {} : { escapeStyle: node.escapeStyle },
+			position
+		};
 		case "html":
 		case "inlineCode": return {
 			type: node.type,
@@ -263,7 +268,7 @@ const admittedFields = {
 	tableCell: [],
 	break: [],
 	thematicBreak: [],
-	text: ["value"],
+	text: ["value", "escapeStyle"],
 	html: ["value"],
 	inlineCode: ["value"],
 	heading: ["depth"],
@@ -434,6 +439,12 @@ var Mdast = class Mdast {
 	* admits spec mdast and nothing else) but it is silent, which is why it is
 	* called out here.
 	*
+	* **One exception: `escapeStyle` on a `text` node is admitted.** It
+	* records no source spelling; it is the caller's instruction to the
+	* emitter (`"literal"` writes the value verbatim, see {@link Text}), so a
+	* plain tree built for `Markdown.stringify` can carry it straight in. A
+	* value outside `"canonical" | "literal"` fails the decode typed.
+	*
 	* @param input - A plain mdast tree, typically a `root`.
 	* @returns A `Result` succeeding with the decoded {@link Root}, or
 	*   failing with {@link MdastDecodeError} carrying the structured issue.

package/README.md

@@ -248,7 +248,7 @@ Effect.runPromise(program).then(console.log);
 
 ## mdast interop
 
-The node classes already use mdast's type names and field shapes; `Mdast.toMdast` strips the fidelity fields this package adds — bullet characters, fence style, ATX-versus-setext spelling — and emits plain spec-valid mdast JSON that the remark ecosystem consumes directly, including `mdast-util-to-hast` if you want hast:
+The node classes already use mdast's type names and field shapes; `Mdast.toMdast` strips the fidelity fields this package adds — bullet characters, fence style, ATX-versus-setext spelling — (keeping only a `Text` node's `escapeStyle` when set, see [Literal text](#literal-text)) and emits plain spec-valid mdast JSON that the remark ecosystem consumes directly, including `mdast-util-to-hast` if you want hast:
 
 ```ts
 import { Markdown, Mdast } from "@effected/markdown";
@@ -317,10 +317,41 @@ MDX nodes are part of the same commitment: the serialization choices listed unde
 
 The posture that makes this a non-issue — and the one to adopt if you build trees and assert on their bytes — is to set `fenceChar` on every `Code` node carrying neither a `lang` nor one already, in a post-decode walk that reaches nested nodes. The choice then leaves the emitter entirely and no output depends on a node's neighbours.
 
-Fidelity fields must be set on the **decoded** tree: `Mdast.fromMdast` admits spec mdast and strips everything outside it, so a `fenceChar` placed on a plain mdast tree before admission is silently dropped.
+Fidelity fields must be set on the **decoded** tree: `Mdast.fromMdast` admits spec mdast and strips everything outside it, so a `fenceChar` placed on a plain mdast tree before admission is silently dropped. The one exception is `escapeStyle` on a `text` node, described next.
 
 A node that carries a fidelity field overrides the matching row — `headingStyle`, `markerChar`, `fenceChar`, `delimiter` — which is how a parsed document re-serializes in its author's spelling. The table describes a synthesized node, which is what a test asserting on generated markdown actually holds.
 
+### Literal text
+
+Text escaping is canonical by default: `~0.2.1` in a table cell emits as `\~0.2.1`. A `Text` node carrying `escapeStyle: "literal"` opts out — the caller vouches that the value is already safe markdown, and the emitter writes it verbatim with none of the escaping aimed at inline syntax. That suits generated content such as a dependency table, where `~0.2.1`, `^1.0.0` and `@scope/pkg` should read as written:
+
+```ts
+import { Markdown, Mdast } from "@effected/markdown";
+import { Result } from "effect";
+
+const cell = (value: string) => ({ type: "tableCell", children: [{ type: "text", value, escapeStyle: "literal" }] });
+const tree = Mdast.fromMdastResult({
+  type: "root",
+  children: [{ type: "table", children: [{ type: "tableRow", children: [cell("Range"), cell("~0.2.1 | ^1.0.0")] }] }],
+});
+if (Result.isSuccess(tree)) {
+  console.log(Result.getOrThrow(Markdown.stringifyResult(tree.success)));
+  // | Range | ~0.2.1 \| ^1.0.0 |
+  // | --- | --- |
+}
+```
+
+Escapes that protect the surrounding **block** still apply, because dropping them would corrupt the document rather than add formatting:
+
+- in a table cell, every `|` not already backslash-escaped becomes `\|`;
+- in any container, a value-final `\` is doubled when more content follows it, so it cannot escape that content's first character;
+- in a table cell or heading, a newline becomes a space;
+- in a heading, a trailing `#` run that would read as the closing sequence is escaped;
+- elsewhere, a newline that would form a blank line becomes `&#10;`, leading whitespace at a line start becomes a character reference, and a line start that can open a block is escaped — a `-`, `+` or `*` bullet or an ordered-list marker followed by a space, tab or line end, one to six `#` followed by the same, `>`, a run of three backticks or tildes, a thematic-break or setext run, a `---` or `+++` frontmatter fence, a line of only `-`, `:`, `|` and whitespace, and conservatively any `<`, `[`, `|` or `:` — while text that cannot, such as `~0.2.1`, `1.0.0`, `-x` or `#x`, is written as is;
+- in a tree carrying MDX nodes, `{` and `<` stay escaped.
+
+**A literal value that parses as markdown does not round-trip**: `*a*` emits verbatim and re-parses as emphasis. Keeping the value free of inline syntax is the caller's promise; the emitter does not check it. The parser never sets `escapeStyle`, and `Mdast.fromMdast` admits it on a `text` node — the one fidelity field that crosses that boundary, because it is an instruction to the emitter rather than a record of source spelling — so a plain mdast tree can carry it straight in. The opt-out is additive: a tree that never sets it serializes byte-identically to the canonical form above.
+
 To *normalize* an existing document to different choices, use `MarkdownFormat` with `MarkdownFormattingOptions`. That is the configurable surface; this one deliberately is not.
 
 ## Features

package/index.d.ts

@@ -170,12 +170,61 @@ export type TableAlign = typeof TableAlign.Type;
 declare const Text_base: Schema.Class<Text, Schema.Struct<{
   readonly type: Schema.tag<"text">;
   readonly value: Schema.String;
+  readonly escapeStyle: Schema.optionalKey<Schema.Literals<readonly ["canonical", "literal"]>>;
   readonly position: Schema.withConstructorDefault<typeof Position>;
 }>, {}>;
 /**
  * Text — a run of literal characters, with entity references and backslash
  * escapes already resolved into `value`.
  *
+ * @remarks
+ * `escapeStyle` is an opt-in emitter extra, the one fidelity field the
+ * parser never sets: absent (or `"canonical"`), `Markdown.stringify` escapes
+ * the value canonically so it re-parses to the same text. `"literal"` is the
+ * caller vouching that `value` is already safe markdown source, so the
+ * emitter writes it verbatim and applies no escaping aimed at inline syntax:
+ * `~0.2.1`, `^1.0.0`, `some_pkg` and `@scope/pkg` come out as written, as do
+ * `*`, `_`, `[`, `]`, `~`, `&`, `\`, the backtick and autolink-shaped
+ * text.
+ *
+ * Escaping that defends the containing BLOCK'S structure still applies,
+ * because its absence would corrupt the document around the text rather
+ * than add formatting inside it:
+ *
+ * - in a table cell, every `|` not already backslash-escaped is written
+ *   `\|` (a bare pipe splits the cell);
+ *
+ * - in any container, a value-final `\` is doubled when more content
+ *   follows it (it would otherwise pair with that content's first
+ *   character, escaping an inline-code backtick or freeing a cell's `\|`);
+ *
+ * - in a table cell or heading, a newline becomes a space (neither can
+ *   hold a line break);
+ *
+ * - in a heading, a trailing `#` run that would read as the ATX closing
+ *   sequence is escaped;
+ *
+ * - elsewhere, a newline that would form a blank line (ending the
+ *   paragraph) becomes `&#10;`, and at every line start leading whitespace
+ *   becomes a character reference, and a line start that can open a block
+ *   is escaped: a `-`, `+` or `*` bullet or an ordered-list marker followed
+ *   by a space, tab or line end; one to six `#` followed by the same; `>`;
+ *   three or more backticks or tildes; a thematic-break or setext run
+ *   (`***`, `___`, `==`); a `---` or `+++` frontmatter fence; a line of
+ *   only `-`, `:`, `|` and whitespace; and, conservatively, any `<`, `[`,
+ *   `|` or `:`. Text that cannot open a block there, such as `~0.2.1`,
+ *   `1.0.0`, `-x` or `#x`, is written as is;
+ *
+ * - in a tree carrying MDX nodes, `{` and `<` stay escaped, since MDX reads
+ *   a stray one as a syntax error rather than as text.
+ *
+ * **A literal text whose value parses as markdown does not round-trip to the
+ * same value**: `*a*` emits verbatim and re-parses as emphasis. Keeping the
+ * value free of inline syntax is the caller's promise, not something the
+ * emitter checks. `Mdast.fromMdast` admits this field on a `text` node (it is
+ * the one emitter instruction a plain tree can carry in), and `Mdast.toMdast`
+ * projects it back out when present.
+ *
  * @public
  */
 export declare class Text extends Text_base {}
@@ -1191,6 +1240,13 @@ export declare class Markdown {
    * rows describe a synthesized node, which is the case a test asserting on
    * generated markdown actually has.
    *
+   * **Text escaping is canonical unless a `Text` node opts out.** A `Text`
+   * carrying `escapeStyle: "literal"` is emitted verbatim apart from the
+   * escapes that protect block structure (a table cell's `|`, a newline in a
+   * single-line container, line-start block openers); its exact set is
+   * documented on {@link Text}. The opt-out is additive: a tree that never
+   * sets it serializes byte-identically to the canonical form.
+   *
    * **Representability wins over the table.** The canonical form never emits
    * text that would re-parse as something else, so a row yields where the two
    * conflict. The case that reaches a consumer is the indented code block: an
@@ -1246,7 +1302,9 @@ export declare class Markdown {
    * admits spec mdast and strips everything outside it, so a `fenceChar`,
    * `headingStyle`, `markerChar` or `delimiter` placed on a plain mdast tree
    * before admission is silently dropped and the canonical default applies.
-   * Set them on the nodes that come back.
+   * Set them on the nodes that come back. The exception is `Text`'s
+   * `escapeStyle`, an emitter instruction rather than a source spelling,
+   * which the boundary admits.
    *
    * @param root - The document tree to serialize.
    * @returns An `Effect` that succeeds with markdown source, or fails with
@@ -2778,6 +2836,12 @@ export declare class Mdast {
    * admits spec mdast and nothing else) but it is silent, which is why it is
    * called out here.
    *
+   * **One exception: `escapeStyle` on a `text` node is admitted.** It
+   * records no source spelling; it is the caller's instruction to the
+   * emitter (`"literal"` writes the value verbatim, see {@link Text}), so a
+   * plain tree built for `Markdown.stringify` can carry it straight in. A
+   * value outside `"canonical" | "literal"` fails the decode typed.
+   *
    * @param input - A plain mdast tree, typically a `root`.
    * @returns A `Result` succeeding with the decoded {@link Root}, or
    *   failing with {@link MdastDecodeError} carrying the structured issue.

package/internal/stringify.js

@@ -184,6 +184,132 @@ const escapeText = (value, context, atLineStart, mdx, followingText, nonTextFoll
 		atLineStart: lineStart
 	};
 };
+/** Space or tab — the whitespace a list marker or ATX run must be followed by. */
+const isSpaceOrTab = (char) => char === " " || char === "	";
+/** Every character of `line` is `marker` or space/tab — a thematic-break or setext-shaped run. */
+const isRunOf = (line, marker) => line.length > 0 && [...line].every((char) => char === marker || char === " " || char === "	");
+/** Only the characters a GFM delimiter row is made of (re2c's `spacechar` included). */
+const reDelimiterRowShaped = /^[-:| \t\v\f]+$/;
+/**
+* Whether a literal line starting with `line` can open a block, so its first
+* character must be escaped. `line` is the text known to be on the line —
+* the value up to its next newline, then the following text siblings — and
+* `open` says the line may carry more than that (no newline ended it).
+* Beyond the known text sits either the end of the line or content this
+* function cannot see (a non-text sibling, a parent's closing delimiter), so
+* a marker that needs "space, tab or end of line" after it is escaped when
+* it reaches the end of `line`. Every test errs toward escaping: an extra
+* backslash is inert, a missing one changes the document.
+*/
+const literalLineOpensBlock = (line, open) => {
+	const char = line[0];
+	switch (char) {
+		case ">":
+		case "<":
+		case "[":
+		case "|":
+		case ":": return true;
+		case "#": {
+			const run = /^#{1,6}/.exec(line)?.[0] ?? "";
+			return run.length > 0 && (run.length === line.length || isSpaceOrTab(line[run.length]));
+		}
+		case "`":
+		case "~": return line.startsWith(char.repeat(3)) || open && [...line].every((each) => each === char);
+		case "=": return /^=+[ \t]*$/.test(line);
+		case "+":
+		case "-":
+		case "*":
+		case "_": {
+			const bullet = char !== "_" && (line.length === 1 || isSpaceOrTab(line[1]));
+			const thematic = char !== "+" && isRunOf(line, char) && (open || [...line].filter((each) => each === char).length >= 3);
+			const frontmatter = (char === "-" || char === "+") && line.startsWith(char.repeat(3));
+			const dashLine = char === "-" && reDelimiterRowShaped.test(line);
+			return bullet || thematic || frontmatter || dashLine;
+		}
+		default: return false;
+	}
+};
+/** ASCII punctuation: what a backslash escapes, so `\x` is one cell character. */
+const reAsciiPunctuation = /[!-/:-@[-`{-~]/;
+const literalText = (value, context, atLineStart, mdx, followingText, nonTextFollows) => {
+	let out = "";
+	let lineStart = atLineStart;
+	for (let index = 0; index < value.length; index += 1) {
+		const char = value[index];
+		if (char === "\n") {
+			const adjacent = value[index - 1] === "\n" || value[index + 1] === "\n";
+			if (context.singleLine) {
+				out += " ";
+				lineStart = false;
+			} else if (adjacent || lineStart) {
+				out += "&#10;";
+				lineStart = false;
+			} else {
+				out += "\n";
+				lineStart = true;
+			}
+			continue;
+		}
+		if (lineStart) {
+			lineStart = false;
+			if (char === "	") {
+				out += "&#9;";
+				continue;
+			}
+			if (char === " ") {
+				out += "&#32;";
+				continue;
+			}
+			const lineEnd = value.indexOf("\n", index);
+			let line = value.slice(index, lineEnd === -1 ? void 0 : lineEnd);
+			let open = false;
+			if (lineEnd === -1) {
+				const followingEnd = followingText.indexOf("\n");
+				line += followingText.slice(0, followingEnd === -1 ? void 0 : followingEnd);
+				open = followingEnd === -1;
+			}
+			const ordered = ORDERED_MARKER.exec(line);
+			if (ordered !== null) {
+				const after = line[ordered[0].length];
+				if (after === void 0 || isSpaceOrTab(after)) {
+					out += `${ordered[0].slice(0, -1)}\\${ordered[0].slice(-1)}`;
+					index += ordered[0].length - 1;
+					continue;
+				}
+			} else if (literalLineOpensBlock(line, open)) {
+				out += `\\${char}`;
+				continue;
+			}
+		}
+		if (char === "\\") {
+			const next = value[index + 1];
+			if (next !== void 0 && reAsciiPunctuation.test(next)) {
+				out += `\\${next}`;
+				index += 1;
+				continue;
+			}
+			out += next === void 0 && (followingText !== "" || nonTextFollows) ? "\\\\" : "\\";
+			continue;
+		}
+		if (char === "|" && context.inTable) {
+			out += "\\|";
+			continue;
+		}
+		if (mdx && (char === "{" || char === "<")) {
+			out += `\\${char}`;
+			continue;
+		}
+		if (context.inHeading && char === "#" && isClosingHashRun(value, index, followingText, nonTextFollows)) {
+			out += "\\#";
+			continue;
+		}
+		out += char;
+	}
+	return {
+		text: out,
+		atLineStart: lineStart
+	};
+};
 /** Pick the effective emphasis marker, flipping away from a forbidden char. */
 const emphasisMarker = (fidelity, forbidden) => {
 	const chosen = fidelity ?? "*";
@@ -260,7 +386,7 @@ const serializeInlines = (children, context, state, startsLine, junctionGuard) =
 					followingText += children[next].value;
 					next += 1;
 				}
-				const escaped = escapeText(child.value, context, atLineStart, state.mdx, followingText, next < children.length);
+				const escaped = (child.escapeStyle === "literal" ? literalText : escapeText)(child.value, context, atLineStart, state.mdx, followingText, next < children.length);
 				out += escaped.text;
 				atLineStart = escaped.atLineStart;
 				break;
@@ -509,13 +635,30 @@ const alignCell = (align) => {
 		default: return "---";
 	}
 };
+/**
+* Escape every cell pipe the GFM cell splitter would read as a column
+* boundary. The splitter consumes a backslash and the punctuation after it as
+* one character, so `\\` is an escaped backslash: a pipe is escaped already
+* iff the run of backslashes directly before it has odd length, and needs a
+* `\` iff that run is even, zero included.
+*/
+const escapeCellPipes = (content) => {
+	let out = "";
+	let backslashes = 0;
+	for (const char of content) {
+		if (char === "|" && backslashes % 2 === 0) out += "\\";
+		backslashes = char === "\\" ? backslashes + 1 : 0;
+		out += char;
+	}
+	return out;
+};
 const serializeTable = (table, state) => {
 	const columnCount = Math.max(1, ...table.children.map((row) => row.children.length));
 	const rows = table.children.map((row) => {
 		guard(state, row);
 		const cells = row.children.map((cell) => {
 			guard(state, cell);
-			const content = serializeInlines(cell.children, CELL_CONTEXT, state, false).replace(/(?<!\\)\|/g, "\\|");
+			const content = escapeCellPipes(serializeInlines(cell.children, CELL_CONTEXT, state, false));
 			unguard(state);
 			return content;
 		});

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@effected/markdown",
-	"version": "0.12.1",
+	"version": "0.14.0",
 	"private": false,
 	"description": "CommonMark and GFM markdown parse, edit and transform schemas for Effect",
 	"keywords": [
@@ -39,10 +39,10 @@
 		"./package.json": "./package.json"
 	},
 	"peerDependencies": {
-		"@effected/jsonc": "^0.13.0",
-		"@effected/toml": "^0.9.0",
-		"@effected/yaml": "^0.17.0",
-		"effect": "4.0.0-rc.117"
+		"@effected/jsonc": "^0.14.0",
+		"@effected/toml": "^0.10.0",
+		"@effected/yaml": "^0.18.0",
+		"effect": "4.0.0-rc.118"
 	},
 	"peerDependenciesMeta": {
 		"@effected/jsonc": {