@vibefield/rosetta-room 0.0.0-stage → 0.1.0

package/dist/components/file-viewer/syntax/languages.js

@@ -0,0 +1,115 @@
+// Which language a file is in, and what to call it.
+//
+// Extension only — never content sniffing. A viewer that guesses the language
+// from the bytes is right most of the time and confidently wrong the rest,
+// and the name it prints is the reader's only way to tell which happened. The
+// file name is a claim its author made; this table just reads it.
+import * as grammars from "./grammars.js";
+import { tokenize } from "./tokenize.js";
+import { tokenizeConfig } from "./tokenize-config.js";
+import { tokenizeCss } from "./tokenize-css.js";
+import { tokenizeMarkup } from "./tokenize-markup.js";
+const withGrammar = (id, label, grammar) => ({
+    id,
+    label,
+    tokenize: (text) => tokenize(text, grammar),
+});
+const language = (id, label, run) => ({
+    id,
+    label,
+    tokenize: run,
+});
+export const PLAIN_TEXT = withGrammar("text", "Text", grammars.plain);
+const TYPESCRIPT = withGrammar("typescript", "TypeScript", grammars.typescript);
+const JAVASCRIPT = withGrammar("javascript", "JavaScript", grammars.typescript);
+const RUST = withGrammar("rust", "Rust", grammars.rust);
+const PYTHON = withGrammar("python", "Python", grammars.python);
+const GO = withGrammar("go", "Go", grammars.go);
+const C = withGrammar("c", "C", grammars.c);
+const CPP = withGrammar("cpp", "C++", grammars.c);
+const JAVA = withGrammar("java", "Java", grammars.java);
+const KOTLIN = withGrammar("kotlin", "Kotlin", grammars.java);
+const SWIFT = withGrammar("swift", "Swift", grammars.swift);
+const RUBY = withGrammar("ruby", "Ruby", grammars.ruby);
+const PHP = withGrammar("php", "PHP", grammars.php);
+const SHELL = withGrammar("shell", "Shell", grammars.shell);
+const SQL = withGrammar("sql", "SQL", grammars.sql);
+const LUA = withGrammar("lua", "Lua", grammars.lua);
+const JSON_LANG = withGrammar("json", "JSON", grammars.json);
+const CSS = language("css", "CSS", tokenizeCss);
+const SCSS = language("scss", "SCSS", tokenizeCss);
+const HTML = language("html", "HTML", tokenizeMarkup);
+const XML = language("xml", "XML", tokenizeMarkup);
+const SVG = language("svg", "SVG", tokenizeMarkup);
+const YAML = language("yaml", "YAML", tokenizeConfig);
+const TOML = language("toml", "TOML", tokenizeConfig);
+const INI = language("ini", "INI", tokenizeConfig);
+/** Markdown read as its own source: the renderer is the markdown view, and
+ * this is what the `source` switch drops back to. */
+const MARKDOWN = language("markdown", "Markdown", (text) => tokenize(text, grammars.plain));
+const BY_EXTENSION = new Map([
+    ["ts", TYPESCRIPT], ["tsx", TYPESCRIPT], ["mts", TYPESCRIPT], ["cts", TYPESCRIPT],
+    ["js", JAVASCRIPT], ["jsx", JAVASCRIPT], ["mjs", JAVASCRIPT], ["cjs", JAVASCRIPT],
+    ["rs", RUST],
+    ["py", PYTHON], ["pyi", PYTHON], ["pyw", PYTHON],
+    ["go", GO],
+    ["c", C], ["h", C],
+    ["cc", CPP], ["cpp", CPP], ["cxx", CPP], ["hpp", CPP], ["hh", CPP], ["hxx", CPP],
+    ["java", JAVA],
+    ["kt", KOTLIN], ["kts", KOTLIN],
+    ["swift", SWIFT],
+    ["rb", RUBY], ["rake", RUBY], ["gemspec", RUBY],
+    ["php", PHP],
+    ["sh", SHELL], ["bash", SHELL], ["zsh", SHELL], ["fish", SHELL], ["ksh", SHELL],
+    ["sql", SQL],
+    ["lua", LUA],
+    ["json", JSON_LANG], ["jsonc", JSON_LANG], ["json5", JSON_LANG], ["webmanifest", JSON_LANG],
+    ["css", CSS],
+    ["scss", SCSS], ["sass", SCSS], ["less", SCSS],
+    ["html", HTML], ["htm", HTML], ["vue", HTML], ["svelte", HTML],
+    ["xml", XML], ["xsl", XML], ["xsd", XML], ["plist", XML], ["storyboard", XML], ["xib", XML],
+    ["svg", SVG],
+    ["yaml", YAML], ["yml", YAML],
+    ["toml", TOML], ["lock", TOML],
+    ["ini", INI], ["cfg", INI], ["conf", INI], ["properties", INI], ["editorconfig", INI],
+    ["md", MARKDOWN], ["markdown", MARKDOWN], ["mdx", MARKDOWN],
+    ["txt", PLAIN_TEXT], ["log", PLAIN_TEXT], ["csv", PLAIN_TEXT], ["tsv", PLAIN_TEXT],
+]);
+/** Files whose name carries the language and whose extension does not. */
+const BY_NAME = new Map([
+    ["Makefile", language("make", "Makefile", (text) => tokenize(text, grammars.shell))],
+    ["Dockerfile", language("dockerfile", "Dockerfile", (text) => tokenize(text, grammars.shell))],
+    ["Gemfile", RUBY],
+    ["Rakefile", RUBY],
+    ["Podfile", RUBY],
+    ["Cargo.lock", TOML],
+    [".gitignore", language("gitignore", "Ignore rules", (text) => tokenize(text, grammars.shell))],
+    [".gitattributes", PLAIN_TEXT],
+    [".npmrc", INI],
+    [".env", SHELL],
+    ["LICENSE", PLAIN_TEXT],
+]);
+/** The last dot-segment of a name, lowercased. `""` when there is no dot. */
+export function extensionOf(name) {
+    const dot = name.lastIndexOf(".");
+    return dot <= 0 ? "" : name.slice(dot + 1).toLowerCase();
+}
+/** The language a file name claims, or `null` when no entry matches — which
+ * is a different answer from "plain text", and the caller decides what to do
+ * with the difference. */
+export function languageFor(name) {
+    const byName = BY_NAME.get(name);
+    if (byName !== undefined)
+        return byName;
+    // `.env.local`, `.eslintrc.json` — the leading dot is part of the name, so
+    // the extension search has to start after it.
+    const stem = name.startsWith(".") ? name.slice(1) : name;
+    const dot = stem.indexOf(".");
+    if (dot !== -1) {
+        const byPrefix = BY_NAME.get(`.${stem.slice(0, dot)}`);
+        if (byPrefix !== undefined && BY_EXTENSION.get(extensionOf(name)) === undefined) {
+            return byPrefix;
+        }
+    }
+    return BY_EXTENSION.get(extensionOf(name)) ?? null;
+}

package/dist/components/file-viewer/syntax/tokenize-config.d.ts

@@ -0,0 +1,2 @@
+import type { Token } from "./types.js";
+export declare function tokenizeConfig(text: string): readonly Token[];

package/dist/components/file-viewer/syntax/tokenize-config.js

@@ -0,0 +1,132 @@
+// YAML, TOML and INI. These are line-shaped languages: what a token means
+// depends on where it sits in its line and almost never on the line before,
+// so the lexer is a per-line reader rather than a character machine.
+//
+// The one exception is YAML's block scalar (`key: |`), whose body is prose
+// that must not be lexed as config. That is carried across lines by
+// indentation, which is the only state this file keeps.
+const KEY = /(?:"[^"\n]*"|'[^'\n]*'|\[\[?[^\]\n]*\]\]?|[A-Za-z0-9_.$/-]+)(?=[ \t]*[:=])/y;
+const SECTION = /\[\[?[^\]\n]*\]\]?/y;
+const STRING = /"""[\s\S]*?"""|'''[\s\S]*?'''|"(?:[^"\\\n]|\\.)*"|'[^'\n]*'/y;
+const NUMBER = /[+-]?(?:0[xXbBoO][0-9a-fA-F_]+|\d[\d_]*(?:\.\d[\d_]*)?(?:[eE][+-]?\d+)?)/y;
+const DATE = /\d{4}-\d{2}-\d{2}(?:[Tt ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})?)?/y;
+const ANCHOR = /[&*][A-Za-z0-9_-]+/y;
+const WORD = /[A-Za-z_][A-Za-z0-9_.-]*/y;
+const CONSTANTS = new Set([
+    "true", "false", "null", "yes", "no", "on", "off", "True", "False", "Null", "None",
+    "TRUE", "FALSE", "NULL", "~",
+]);
+export function tokenizeConfig(text) {
+    const out = [];
+    const push = (kind, value) => {
+        if (value.length > 0)
+            out.push({ kind, text: value });
+    };
+    /** Indentation of the block scalar currently being consumed as prose, or
+     * null when there is none. */
+    let literalIndent = null;
+    const lines = text.split("\n");
+    lines.forEach((line, lineIndex) => {
+        if (lineIndex > 0)
+            push("plain", "\n");
+        const indent = line.length - line.trimStart().length;
+        if (literalIndent !== null) {
+            if (line.trim() === "" || indent > literalIndent) {
+                push("string", line);
+                return;
+            }
+            literalIndent = null;
+        }
+        let index = 0;
+        const match = (pattern) => {
+            pattern.lastIndex = index;
+            const found = pattern.exec(line);
+            return found === null ? null : found[0];
+        };
+        const emit = (kind, value) => {
+            push(kind, value);
+            index += value.length;
+        };
+        const skipSpace = () => {
+            const space = match(/[ \t]+/y);
+            if (space !== null)
+                emit("plain", space);
+        };
+        skipSpace();
+        if (index >= line.length) {
+            push("plain", line.slice(index));
+            return;
+        }
+        if (line[index] === "#" || line[index] === ";") {
+            emit("comment", line.slice(index));
+            return;
+        }
+        // A list item is a marker plus a value — or plus a key, for a list of
+        // tables, which is why this falls through rather than returning.
+        while (line[index] === "-" && (line[index + 1] === " " || index + 1 === line.length)) {
+            emit("operator", "-");
+            skipSpace();
+        }
+        const section = line[index] === "[" ? match(SECTION) : null;
+        if (section !== null) {
+            emit("type", section);
+            skipSpace();
+            if (line[index] === "#" || line[index] === ";")
+                emit("comment", line.slice(index));
+            return;
+        }
+        const key = match(KEY);
+        if (key !== null) {
+            emit("property", key);
+            skipSpace();
+            emit("operator", line[index] ?? "");
+            skipSpace();
+        }
+        // Whatever is left of the line is a value.
+        while (index < line.length) {
+            const char = line[index];
+            if (char === "#" && (index === 0 || /[ \t]/.test(line[index - 1] ?? " "))) {
+                emit("comment", line.slice(index));
+                break;
+            }
+            if (/[ \t]/.test(char)) {
+                skipSpace();
+                continue;
+            }
+            if ((char === "|" || char === ">") && index === line.length - 1) {
+                emit("operator", char);
+                literalIndent = indent;
+                break;
+            }
+            if (char === '"' || char === "'") {
+                const string = match(STRING);
+                emit("string", string ?? line.slice(index));
+                continue;
+            }
+            if (char === "&" || char === "*") {
+                const anchor = match(ANCHOR);
+                if (anchor !== null) {
+                    emit("type", anchor);
+                    continue;
+                }
+            }
+            const date = match(DATE);
+            if (date !== null) {
+                emit("number", date);
+                continue;
+            }
+            const number = match(NUMBER);
+            if (number !== null) {
+                emit("number", number);
+                continue;
+            }
+            const word = match(WORD);
+            if (word !== null) {
+                emit(CONSTANTS.has(word) ? "constant" : "plain", word);
+                continue;
+            }
+            emit(CONSTANTS.has(char) ? "constant" : "operator", char);
+        }
+    });
+    return out;
+}

package/dist/components/file-viewer/syntax/tokenize-css.d.ts

@@ -0,0 +1,2 @@
+import type { Token } from "./types.js";
+export declare function tokenizeCss(text: string): readonly Token[];

package/dist/components/file-viewer/syntax/tokenize-css.js

@@ -0,0 +1,169 @@
+// CSS, SCSS and Less. A stylesheet has three worlds rather than one — the
+// selector, the property, the value — and the same identifier means a
+// different thing in each. So the lexer carries the one bit of state that
+// tells them apart: whether it is inside a block, and whether it is past the
+// colon.
+const IDENTIFIER = /-{0,2}[A-Za-z_][A-Za-z0-9_-]*/y;
+const AT_RULE = /@[A-Za-z-]+/y;
+const HEX = /#[0-9a-fA-F]{3,8}\b/y;
+const NUMBER = /[+-]?(?:\d+\.?\d*|\.\d+)(?:%|[a-zA-Z]+)?/y;
+const VARIABLE = /[$@][A-Za-z_][A-Za-z0-9_-]*/y;
+const PSEUDO = /::?[A-Za-z-][A-Za-z0-9_-]*/y;
+const ID_SELECTOR = /#[A-Za-z_-][A-Za-z0-9_-]*/y;
+const CLASS_SELECTOR = /\.[A-Za-z_-][A-Za-z0-9_-]*/y;
+const WHITESPACE = /[ \t\r\n]+/y;
+/** Values the language names, as opposed to values an author invents. */
+const VALUE_CONSTANTS = new Set([
+    "auto", "inherit", "initial", "unset", "none", "revert", "currentColor", "transparent",
+    "true", "false", "important",
+]);
+export function tokenizeCss(text) {
+    const out = [];
+    let index = 0;
+    let depth = 0;
+    /** Past the `:` of a declaration, where identifiers are values. */
+    let inValue = false;
+    const push = (kind, value) => {
+        if (value.length === 0)
+            return;
+        out.push({ kind, text: value });
+        index += value.length;
+    };
+    const match = (pattern) => {
+        pattern.lastIndex = index;
+        const found = pattern.exec(text);
+        return found === null ? null : found[0];
+    };
+    const through = (from, closer) => {
+        const at = text.indexOf(closer, from);
+        return at === -1 ? text.slice(index) : text.slice(index, at + closer.length);
+    };
+    const peekNonSpace = (from) => {
+        let i = from;
+        while (i < text.length && /[ \t\r\n]/.test(text[i]))
+            i += 1;
+        return text[i] ?? "";
+    };
+    while (index < text.length) {
+        const char = text[index];
+        if (/[ \t\r\n]/.test(char)) {
+            push("plain", match(WHITESPACE) ?? char);
+            continue;
+        }
+        if (text.startsWith("/*", index)) {
+            push("comment", through(index + 2, "*/"));
+            continue;
+        }
+        // SCSS and Less only; plain CSS has no line comment, and a file that
+        // uses one is not plain CSS.
+        if (text.startsWith("//", index)) {
+            const end = text.indexOf("\n", index);
+            push("comment", text.slice(index, end === -1 ? text.length : end));
+            continue;
+        }
+        if (char === '"' || char === "'") {
+            push("string", through(index + 1, char));
+            continue;
+        }
+        if (char === "{") {
+            push("operator", "{");
+            depth += 1;
+            inValue = false;
+            continue;
+        }
+        if (char === "}") {
+            push("operator", "}");
+            depth = Math.max(0, depth - 1);
+            inValue = false;
+            continue;
+        }
+        if (char === ";") {
+            push("operator", ";");
+            inValue = false;
+            continue;
+        }
+        if (char === ":") {
+            // Inside a block a colon opens a value; in a selector it opens a
+            // pseudo-class, which is part of the name rather than a separator.
+            if (depth > 0 && !inValue) {
+                push("operator", ":");
+                inValue = true;
+                continue;
+            }
+            const pseudo = match(PSEUDO);
+            push(pseudo === null ? "operator" : "constant", pseudo ?? ":");
+            continue;
+        }
+        // A Less variable is a VALUE, so it is the value position that tells it
+        // apart from an at-rule — not the nesting depth, which reads SCSS's
+        // nested `@media` as a variable.
+        if (char === "$" || (char === "@" && inValue)) {
+            const variable = match(VARIABLE);
+            if (variable !== null) {
+                push("constant", variable);
+                continue;
+            }
+        }
+        if (char === "@") {
+            const atRule = match(AT_RULE);
+            if (atRule !== null) {
+                push("keyword", atRule);
+                continue;
+            }
+        }
+        if (char === "!") {
+            const bang = match(/![ \t]*[a-z]+/y);
+            if (bang !== null) {
+                push("keyword", bang);
+                continue;
+            }
+        }
+        if (char === "#") {
+            const hex = match(HEX);
+            if (hex !== null) {
+                push("number", hex);
+                continue;
+            }
+            // Not a colour, so an id selector.
+            const id = match(ID_SELECTOR);
+            push(id === null ? "operator" : "type", id ?? "#");
+            continue;
+        }
+        if (char === ".") {
+            const className = match(CLASS_SELECTOR);
+            if (className !== null) {
+                push("type", className);
+                continue;
+            }
+        }
+        // `-4px` is one negative length, not an operator and a length. `--x` and
+        // `-webkit-` are not numbers, and the digit test is what separates them.
+        if (/[0-9]/.test(char) ||
+            ((char === "." || char === "-") && /[0-9.]/.test(text[index + 1] ?? ""))) {
+            const number = match(NUMBER);
+            if (number !== null) {
+                push("number", number);
+                continue;
+            }
+        }
+        const word = match(IDENTIFIER);
+        if (word !== null) {
+            const next = peekNonSpace(index + word.length);
+            let kind;
+            if (next === "(")
+                kind = "function";
+            else if (word.startsWith("--"))
+                kind = "property";
+            else if (inValue)
+                kind = VALUE_CONSTANTS.has(word) ? "constant" : "plain";
+            else if (depth > 0)
+                kind = "property";
+            else
+                kind = "tag";
+            push(kind, word);
+            continue;
+        }
+        push("operator", char);
+    }
+    return out;
+}

package/dist/components/file-viewer/syntax/tokenize-markup.d.ts

@@ -0,0 +1,2 @@
+import type { Token } from "./types.js";
+export declare function tokenizeMarkup(text: string): readonly Token[];

package/dist/components/file-viewer/syntax/tokenize-markup.js

@@ -0,0 +1,124 @@
+// HTML, XML and SVG. Markup is not the C family wearing different keywords —
+// it has two alternating worlds, text and tag, and a table-driven lexer that
+// pretends otherwise paints attribute values as identifiers.
+//
+// A tag name takes the `tag` kind and an attribute name takes `property`,
+// which share their accents with `keyword` and an object key. They are the
+// same two roles in another syntax, and the room should not teach a reader
+// two palettes for one idea.
+const TAG_NAME = /[A-Za-z_][A-Za-z0-9_:.-]*/y;
+/** Plain `href`, and the framework spellings a real page is full of:
+ * `v-if`, `:class`, `@click`, `#ref`, `[prop]`, `(click)`. */
+const ATTRIBUTE_NAME = /[A-Za-z_@:#$[(][-A-Za-z0-9_@:.$\])]*/y;
+const ENTITY = /&[a-zA-Z][a-zA-Z0-9]*;|&#[0-9]+;|&#x[0-9a-fA-F]+;/y;
+/** Elements whose body is another language, not markup. */
+const RAW_TEXT = new Set(["script", "style"]);
+export function tokenizeMarkup(text) {
+    const out = [];
+    let index = 0;
+    const push = (kind, value) => {
+        if (value.length === 0)
+            return;
+        out.push({ kind, text: value });
+        index += value.length;
+    };
+    /** Everything up to `end`, or the rest of the file when it never arrives. */
+    const through = (from, closer) => {
+        const at = text.indexOf(closer, from);
+        return at === -1 ? text.slice(index) : text.slice(index, at + closer.length);
+    };
+    const match = (pattern) => {
+        pattern.lastIndex = index;
+        const found = pattern.exec(text);
+        return found === null ? null : found[0];
+    };
+    /** Where `<script>`'s body ends. Lowercased lazily: most files have no raw
+     * text in them and should not pay for a second copy of themselves. */
+    let lowered = null;
+    const rawTextEnd = (from, tag) => {
+        lowered ??= text.toLowerCase();
+        const at = lowered.indexOf(`</${tag.toLowerCase()}`, from);
+        return at === -1 ? text.length : at;
+    };
+    /** A quoted attribute value, stopped at the end of the line when the quote
+     * never closes. Losing a wrapped value is a smaller wrong than one typo
+     * painting the rest of the document as a string — which is the same trade
+     * the generic tokenizer already makes for a single-line literal. */
+    const attributeValue = (quote) => {
+        const raw = through(index + 1, quote);
+        const newline = raw.indexOf("\n");
+        return newline === -1 ? raw : raw.slice(0, newline);
+    };
+    while (index < text.length) {
+        if (text.startsWith("<!--", index)) {
+            push("comment", through(index + 4, "-->"));
+            continue;
+        }
+        if (text.startsWith("<![CDATA[", index)) {
+            push("string", through(index + 9, "]]>"));
+            continue;
+        }
+        if (text.startsWith("<!", index) || text.startsWith("<?", index)) {
+            push("keyword", through(index + 2, ">"));
+            continue;
+        }
+        if (text[index] === "<") {
+            const closing = text.startsWith("</", index);
+            push("operator", closing ? "</" : "<");
+            const name = match(TAG_NAME);
+            if (name !== null)
+                push("tag", name);
+            // Inside the tag until `>`: attribute, `=`, value, repeat.
+            while (index < text.length && text[index] !== ">") {
+                const char = text[index];
+                if (char === " " || char === "\t" || char === "\n" || char === "\r") {
+                    push("plain", match(/[ \t\r\n]+/y) ?? char);
+                    continue;
+                }
+                if (char === "/") {
+                    push("operator", "/");
+                    continue;
+                }
+                if (char === "=") {
+                    push("operator", "=");
+                    continue;
+                }
+                if (char === '"' || char === "'") {
+                    push("string", attributeValue(char));
+                    continue;
+                }
+                const attribute = match(ATTRIBUTE_NAME);
+                if (attribute !== null) {
+                    push("property", attribute);
+                    continue;
+                }
+                // An unquoted value, or something this lexer has no name for.
+                push("plain", char);
+            }
+            if (index < text.length) {
+                // `<br/>` closed itself; `<div>` did not. The character before the
+                // `>` is the whole difference.
+                const selfClosing = text[index - 1] === "/";
+                push("operator", ">");
+                // A `<script>` body is JavaScript, and `if (a < b)` inside it is not
+                // the start of a tag. Lexing it as markup used to swallow everything
+                // to the next `>` — including the `</script>` that ends the element.
+                if (!closing && !selfClosing && name !== null && RAW_TEXT.has(name.toLowerCase())) {
+                    push("plain", text.slice(index, rawTextEnd(index, name)));
+                }
+            }
+            continue;
+        }
+        const entity = match(ENTITY);
+        if (entity !== null) {
+            push("constant", entity);
+            continue;
+        }
+        // Text content, in one run up to the next markup character.
+        const next = text.indexOf("<", index + 1);
+        const ampersand = text.indexOf("&", index + 1);
+        const stop = [next, ampersand].filter((at) => at !== -1).sort((a, b) => a - b)[0] ?? text.length;
+        push("plain", text.slice(index, Math.max(stop, index + 1)));
+    }
+    return out;
+}

package/dist/components/file-viewer/syntax/tokenize.d.ts

@@ -0,0 +1,9 @@
+import type { Grammar, Token, TokenLine } from "./types.js";
+export declare function tokenize(text: string, grammar: Grammar): readonly Token[];
+/**
+ * Cut a token stream into lines. Tokens that span newlines — block comments,
+ * template literals, docstrings — are split, and each piece keeps its kind,
+ * which is the whole reason tokenizing runs over the file rather than over
+ * each line in turn.
+ */
+export declare function splitLines(tokens: readonly Token[]): readonly TokenLine[];