@stxt-lang/core 0.11.0 → 0.11.1

package/README.md

@@ -291,6 +291,22 @@ const text = NodeWriter.toSTXT(node, IndentStyle.TABS);
 const doc = NodeWriter.toSTXTDocs(result.getNodes(), IndentStyle.SPACES_4);
 ```
 
+`NodeWriter` re-serializes the tree, so comments and blank lines are gone. To reformat a document
+**keeping everything the author wrote**, use `Formatter`: it rewrites the original text line by
+line — node lines in canonical form, block lines re-indented to their block, comments and blank
+lines kept with their indentation units converted — and reports the syntax errors it met, so the
+caller decides what to do with a document that does not parse. It is the formatter behind
+`stxt format`, the VS Code extension and the playground.
+
+```ts
+import { Formatter, IndentStyle } from '@stxt-lang/core';
+
+const { text, errors } = Formatter.format(source, IndentStyle.TABS);
+if (errors.length === 0) {
+  fs.writeFileSync(file, text);
+}
+```
+
 ## API surface
 
 Everything importable from the package:
@@ -300,7 +316,7 @@ Everything importable from the package:
 - **Extension points** — `Observer`, `Validator`
 - **Schemas** — `Schema`, `SchemaValidator`, `SchemaProvider`, `SchemaProviderMemory`, `SchemaProviderMeta`, `NodeDefinition`, `ChildDefinition`, `TypeRegistry`, `Type`, `transformNodeToSchema`
 - **Templates** — `transformTemplateNodeToSchema`, `TEMPLATE_NAMESPACE`, `TemplateSchemaProviderMemory`, `MetaTemplateSchemaProvider`
-- **Runtime** — `UnifiedSchemaProvider`, `NodeWriter`, `IndentStyle`
+- **Runtime** — `UnifiedSchemaProvider`, `NodeWriter`, `IndentStyle`, `Formatter`, `FormatResult`, `toCanonicalTree`, `toCanonicalJson`
 - **Discovery** — `DiscoveryResolver`, `DiscoveryOptions`, `DiscoveryResult`, `DiscoveryDefinition`, `DiscoveryLevel`, `DiscoveryError`, `DiscoveryFileSystem`, `DiscoveryEntry`, `DiscoveryEnvironment`
 
 ## License

package/out/all.d.ts

@@ -30,6 +30,8 @@ export { ChildDefinition } from "./schema/ChildDefinition";
 export { transformNodeToSchema } from "./schema/SchemaParser";
 export { UnifiedSchemaProvider } from "./runtime/UnifiedSchemaProvider";
 export { NodeWriter, IndentStyle } from "./runtime/NodeWriter";
+export { Formatter } from "./runtime/Formatter";
+export type { FormatResult } from "./runtime/Formatter";
 export { toCanonicalTree, toCanonicalJson } from "./runtime/TreeJson";
 export type { CanonicalDocument, CanonicalNode, CanonicalInlineNode, CanonicalBlockNode } from "./runtime/TreeJson";
 export { transformTemplateNodeToSchema, TEMPLATE_NAMESPACE } from "./template/TemplateParser";

package/out/all.js

@@ -3,7 +3,7 @@
 // Anything that should be consumable by third parties (e.g. the VSCode extension)
 // has to be re-exported from here.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DiscoveryError = exports.DiscoveryResult = exports.DiscoveryResolver = exports.MetaTemplateSchemaProvider = exports.TemplateSchemaProviderMemory = exports.TEMPLATE_NAMESPACE = exports.transformTemplateNodeToSchema = exports.toCanonicalJson = exports.toCanonicalTree = exports.IndentStyle = exports.NodeWriter = exports.UnifiedSchemaProvider = exports.transformNodeToSchema = exports.ChildDefinition = exports.NodeDefinition = exports.TypeRegistry = exports.SchemaProviderMeta = exports.SchemaProviderMemory = exports.SchemaValidator = exports.Schema = exports.RuntimeException = exports.ValidationException = exports.ParseException = exports.StringUtils = exports.parseLine = exports.SPEC_VERSION = exports.Constants = exports.Line = exports.ParseResult = exports.Parser = exports.TextNode = exports.InlineNode = exports.Node = void 0;
+exports.DiscoveryError = exports.DiscoveryResult = exports.DiscoveryResolver = exports.MetaTemplateSchemaProvider = exports.TemplateSchemaProviderMemory = exports.TEMPLATE_NAMESPACE = exports.transformTemplateNodeToSchema = exports.toCanonicalJson = exports.toCanonicalTree = exports.Formatter = exports.IndentStyle = exports.NodeWriter = exports.UnifiedSchemaProvider = exports.transformNodeToSchema = exports.ChildDefinition = exports.NodeDefinition = exports.TypeRegistry = exports.SchemaProviderMeta = exports.SchemaProviderMemory = exports.SchemaValidator = exports.Schema = exports.RuntimeException = exports.ValidationException = exports.ParseException = exports.StringUtils = exports.parseLine = exports.SPEC_VERSION = exports.Constants = exports.Line = exports.ParseResult = exports.Parser = exports.TextNode = exports.InlineNode = exports.Node = void 0;
 var Node_1 = require("./core/Node");
 Object.defineProperty(exports, "Node", { enumerable: true, get: function () { return Node_1.Node; } });
 var InlineNode_1 = require("./core/InlineNode");
@@ -56,6 +56,8 @@ Object.defineProperty(exports, "UnifiedSchemaProvider", { enumerable: true, get:
 var NodeWriter_1 = require("./runtime/NodeWriter");
 Object.defineProperty(exports, "NodeWriter", { enumerable: true, get: function () { return NodeWriter_1.NodeWriter; } });
 Object.defineProperty(exports, "IndentStyle", { enumerable: true, get: function () { return NodeWriter_1.IndentStyle; } });
+var Formatter_1 = require("./runtime/Formatter");
+Object.defineProperty(exports, "Formatter", { enumerable: true, get: function () { return Formatter_1.Formatter; } });
 var TreeJson_1 = require("./runtime/TreeJson");
 Object.defineProperty(exports, "toCanonicalTree", { enumerable: true, get: function () { return TreeJson_1.toCanonicalTree; } });
 Object.defineProperty(exports, "toCanonicalJson", { enumerable: true, get: function () { return TreeJson_1.toCanonicalJson; } });

package/out/runtime/Formatter.d.ts

@@ -0,0 +1,104 @@
+import { ParseException } from "../exceptions/ParseException";
+import { IndentStyle } from "./NodeWriter";
+/** The outcome of {@link Formatter.format}. */
+export interface FormatResult {
+    /**
+     * The formatted document: the same lines as the source, in the same order, with the same
+     * line ending (CRLF is kept) and with a final newline only where the source had one.
+     */
+    text: string;
+    /**
+     * Syntax errors found while parsing, in line order; empty when the document parses. A line
+     * the parse tree does not describe because of an error is only converted by indentation
+     * units and right-trimmed (see {@link Formatter}): formatting never repairs a document, and
+     * whether a document with errors should be reformatted at all is the caller's decision.
+     */
+    errors: ParseException[];
+}
+/**
+ * Reformats an STXT document **line by line, over the original text**, so that nothing the
+ * parse tree does not hold — comments, blank lines, the exact content of text blocks — is lost.
+ * This is what distinguishes it from {@link NodeWriter}, which re-serializes the tree and
+ * therefore drops comments and blank lines.
+ *
+ * The rules, the same for every tool of the ecosystem (the CLI's `stxt format`, the VS Code
+ * extension's formatter and the playground's re-indentation all delegate here):
+ *
+ * - A line that **opens a node** is rendered in canonical form: the indentation of its level in
+ *   the requested style, the name as parsed, the namespace only where the source wrote it (a
+ *   child repeating its parent's namespace is redundant but legal, and dropping it would be an
+ *   edit, not a reformat), `: value` with exactly one space — or a bare `:` when there is no
+ *   value, so container nodes do not end in a stray space — or ` >>` for a block.
+ * - A **text line of a block** gets the indentation of the block (its level plus one) in the
+ *   requested style, followed by its content; any indentation the line had beyond the block's
+ *   is content (STXT-SPEC §10.2, relative indentation is preserved) and is kept exactly. A
+ *   blank line of the block is `""` in the content whatever it looks like in the source
+ *   (STXT-SPEC §10.3), so it is written with the indentation of the block too: the block reads
+ *   as one piece and, at the end of the file, the line is not lost — an empty last line would
+ *   be indistinguishable from the final line ending.
+ * - Every **other line** — a comment, a blank line outside a block, or a line the parse tree
+ *   does not describe because of a syntax error — is kept as the author wrote it, except that
+ *   its trailing blanks are removed and the whole indentation units at its start are converted
+ *   one for one to the requested style (a tab or four spaces in either style count as a unit;
+ *   whatever follows the last whole unit, including a remainder that is not a whole unit, is
+ *   kept as it is). STXT-SPEC §9 validates the indentation of a comment like a node's, so in a
+ *   document that parses every comment has a whole number of units and comes out fully in the
+ *   new style; the remainder only survives in documents with errors, which this conversion
+ *   neither repairs nor hides.
+ *
+ * The result is idempotent, round-trips between the two styles, and produces the same canonical
+ * tree (STXT-TREE-SPEC) as the source. The document is parsed without any schema: formatting
+ * has nothing to do with validation.
+ */
+export declare class Formatter {
+    private constructor();
+    /**
+     * Formats a document.
+     *
+     * @param text the document.
+     * @param style indentation style to format with; tabs by default.
+     * @returns the formatted text and the syntax errors found; see {@link FormatResult}.
+     */
+    static format(text: string, style?: IndentStyle): FormatResult;
+    /**
+     * Formats one source line.
+     *
+     * @param line the line, without its line ending.
+     * @param lineNumber its line number, 1-indexed as the parser counts them.
+     * @param style indentation style to format with.
+     * @param sourceLines the parse of the document seen as source lines.
+     * @returns the formatted line.
+     */
+    private static formatLine;
+    /**
+     * Renders the line that opens a node in its canonical form.
+     *
+     * @param node the node the line opens.
+     * @param line the source line, used only to tell whether it spelled the namespace out.
+     * @param style indentation style to format with.
+     * @returns the formatted line.
+     */
+    private static renderNode;
+    /**
+     * Converts the whole indentation units at the start of a line to the requested style and
+     * keeps the rest of the line, remainder included.
+     *
+     * @param line the line, without trailing blanks.
+     * @param style indentation style to convert to.
+     * @returns the line with its indentation units converted.
+     */
+    private static convertUnits;
+    /**
+     * @param line a line.
+     * @param position a position in it.
+     * @returns the length of the whole indentation unit — a tab or four spaces — that starts at
+     *          `position`, or 0 if none does.
+     */
+    private static unitAt;
+    /**
+     * @param level indentation level to produce.
+     * @param style indentation style to produce it in.
+     * @returns the indentation of that level.
+     */
+    private static indent;
+}

package/out/runtime/Formatter.js

@@ -0,0 +1,184 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Formatter = void 0;
+const InlineNode_1 = require("../core/InlineNode");
+const Parser_1 = require("../core/Parser");
+const StringUtils_1 = require("../core/StringUtils");
+const TextNode_1 = require("../core/TextNode");
+const NodeWriter_1 = require("./NodeWriter");
+/**
+ * Reformats an STXT document **line by line, over the original text**, so that nothing the
+ * parse tree does not hold — comments, blank lines, the exact content of text blocks — is lost.
+ * This is what distinguishes it from {@link NodeWriter}, which re-serializes the tree and
+ * therefore drops comments and blank lines.
+ *
+ * The rules, the same for every tool of the ecosystem (the CLI's `stxt format`, the VS Code
+ * extension's formatter and the playground's re-indentation all delegate here):
+ *
+ * - A line that **opens a node** is rendered in canonical form: the indentation of its level in
+ *   the requested style, the name as parsed, the namespace only where the source wrote it (a
+ *   child repeating its parent's namespace is redundant but legal, and dropping it would be an
+ *   edit, not a reformat), `: value` with exactly one space — or a bare `:` when there is no
+ *   value, so container nodes do not end in a stray space — or ` >>` for a block.
+ * - A **text line of a block** gets the indentation of the block (its level plus one) in the
+ *   requested style, followed by its content; any indentation the line had beyond the block's
+ *   is content (STXT-SPEC §10.2, relative indentation is preserved) and is kept exactly. A
+ *   blank line of the block is `""` in the content whatever it looks like in the source
+ *   (STXT-SPEC §10.3), so it is written with the indentation of the block too: the block reads
+ *   as one piece and, at the end of the file, the line is not lost — an empty last line would
+ *   be indistinguishable from the final line ending.
+ * - Every **other line** — a comment, a blank line outside a block, or a line the parse tree
+ *   does not describe because of a syntax error — is kept as the author wrote it, except that
+ *   its trailing blanks are removed and the whole indentation units at its start are converted
+ *   one for one to the requested style (a tab or four spaces in either style count as a unit;
+ *   whatever follows the last whole unit, including a remainder that is not a whole unit, is
+ *   kept as it is). STXT-SPEC §9 validates the indentation of a comment like a node's, so in a
+ *   document that parses every comment has a whole number of units and comes out fully in the
+ *   new style; the remainder only survives in documents with errors, which this conversion
+ *   neither repairs nor hides.
+ *
+ * The result is idempotent, round-trips between the two styles, and produces the same canonical
+ * tree (STXT-TREE-SPEC) as the source. The document is parsed without any schema: formatting
+ * has nothing to do with validation.
+ */
+class Formatter {
+    constructor() { }
+    /**
+     * Formats a document.
+     *
+     * @param text the document.
+     * @param style indentation style to format with; tabs by default.
+     * @returns the formatted text and the syntax errors found; see {@link FormatResult}.
+     */
+    static format(text, style = NodeWriter_1.IndentStyle.TABS) {
+        const sourceLines = new SourceLines();
+        const parser = new Parser_1.Parser();
+        parser.registerObserver(sourceLines);
+        const result = parser.parseResult(text);
+        const eol = text.includes("\r\n") ? "\r\n" : "\n";
+        const formatted = text
+            .split(/\r?\n/)
+            .map((line, index) => Formatter.formatLine(line, index + 1, style, sourceLines))
+            .join(eol);
+        return { text: formatted, errors: result.getErrors() };
+    }
+    /**
+     * Formats one source line.
+     *
+     * @param line the line, without its line ending.
+     * @param lineNumber its line number, 1-indexed as the parser counts them.
+     * @param style indentation style to format with.
+     * @param sourceLines the parse of the document seen as source lines.
+     * @returns the formatted line.
+     */
+    static formatLine(line, lineNumber, style, sourceLines) {
+        const node = sourceLines.nodeAt(lineNumber);
+        if (node) {
+            return Formatter.renderNode(node, line, style);
+        }
+        const text = sourceLines.textAt(lineNumber);
+        if (text) {
+            return Formatter.indent(text.node.getLevel() + 1, style) + text.line.content;
+        }
+        return Formatter.convertUnits(StringUtils_1.StringUtils.rightTrim(line), style);
+    }
+    /**
+     * Renders the line that opens a node in its canonical form.
+     *
+     * @param node the node the line opens.
+     * @param line the source line, used only to tell whether it spelled the namespace out.
+     * @param style indentation style to format with.
+     * @returns the formatted line.
+     */
+    static renderNode(node, line, style) {
+        const indent = Formatter.indent(node.getLevel(), style);
+        const head = node instanceof InlineNode_1.InlineNode ? line.substring(0, line.indexOf(":")) : line;
+        const name = head.includes("(")
+            ? `${node.getName()} (${node.getNamespace()})`
+            : node.getName();
+        if (node instanceof TextNode_1.TextNode) {
+            return `${indent}${name} >>`;
+        }
+        const value = node.getValue();
+        return value.length > 0 ? `${indent}${name}: ${value}` : `${indent}${name}:`;
+    }
+    /**
+     * Converts the whole indentation units at the start of a line to the requested style and
+     * keeps the rest of the line, remainder included.
+     *
+     * @param line the line, without trailing blanks.
+     * @param style indentation style to convert to.
+     * @returns the line with its indentation units converted.
+     */
+    static convertUnits(line, style) {
+        let consumed = 0;
+        let units = 0;
+        let unit = Formatter.unitAt(line, consumed);
+        while (unit > 0) {
+            consumed += unit;
+            units++;
+            unit = Formatter.unitAt(line, consumed);
+        }
+        return units === 0 ? line : Formatter.indent(units, style) + line.substring(consumed);
+    }
+    /**
+     * @param line a line.
+     * @param position a position in it.
+     * @returns the length of the whole indentation unit — a tab or four spaces — that starts at
+     *          `position`, or 0 if none does.
+     */
+    static unitAt(line, position) {
+        if (line.startsWith("\t", position)) {
+            return 1;
+        }
+        return line.startsWith("    ", position) ? 4 : 0;
+    }
+    /**
+     * @param level indentation level to produce.
+     * @param style indentation style to produce it in.
+     * @returns the indentation of that level.
+     */
+    static indent(level, style) {
+        return (style === NodeWriter_1.IndentStyle.SPACES_4 ? "    " : "\t").repeat(level);
+    }
+}
+exports.Formatter = Formatter;
+/**
+ * The parse of a document seen as source lines: which line opened which node, and which line is
+ * a text line of which block. It is what lets the formatter rewrite the lines the parse tree
+ * describes and leave every other line as the author wrote it.
+ */
+class SourceLines {
+    constructor() {
+        this.nodeByLine = new Map();
+        this.textByLine = new Map();
+    }
+    onCreate(node) {
+        this.nodeByLine.set(node.getLine(), node);
+    }
+    onFinish() {
+        // Formatting only needs to know where each node started.
+    }
+    onComment() {
+        // Comment lines need no bookkeeping: every line that opens no node is treated alike.
+    }
+    onTextLine(node, lineNumber, lineString, line) {
+        this.textByLine.set(lineNumber, { node, line });
+    }
+    /**
+     * @param lineNumber line number, 1-indexed.
+     * @returns the node this line opened, or undefined if it opened none.
+     */
+    nodeAt(lineNumber) {
+        return this.nodeByLine.get(lineNumber);
+    }
+    /**
+     * @param lineNumber line number, 1-indexed.
+     * @returns the block node this line is text of and the line already split into indentation
+     *          and content, or undefined if the line is not text of a block.
+     */
+    textAt(lineNumber) {
+        return this.textByLine.get(lineNumber);
+    }
+}
+//# sourceMappingURL=Formatter.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stxt-lang/core",
-  "version": "0.11.0",
+  "version": "0.11.1",
   "description": "Parser and schema validator for STXT, an indentation-based structured-text format.",
   "main": "out/all.js",
   "types": "out/all.d.ts",