@clickhouse/datatype-parser 0.1.0

package/dist/parser.js

@@ -0,0 +1,460 @@
+/// A faithful port of ClickHouse's `ParserDataType::parseImpl`
+/// (src/Parsers/ParserDataType.cpp) onto the self-contained AST in `ast.ts`.
+/// The control flow deliberately tracks the original: identifier + SQL-standard
+/// multi-word aliases, the Enum and Tuple special cases, then the generic
+/// parametric-argument loop. AggregateFunction/SimpleAggregateFunction and the
+/// JSON object-argument syntax are reported as unsupported (see parser.h).
+///
+/// This is the TypeScript port of the C++ `chdt/parser.cpp`.
+import { makeNode, NodeKind } from "./ast.js";
+import { tokenize, TokenType } from "./lexer.js";
+function toUpper(s) {
+    let r = "";
+    for (let i = 0; i < s.length; ++i) {
+        const c = s[i];
+        if (c >= "a" && c <= "z")
+            r += String.fromCharCode(c.charCodeAt(0) - "a".charCodeAt(0) + "A".charCodeAt(0));
+        else
+            r += c;
+    }
+    return r;
+}
+function toLower(s) {
+    let r = "";
+    for (let i = 0; i < s.length; ++i) {
+        const c = s[i];
+        if (c >= "A" && c <= "Z")
+            r += String.fromCharCode(c.charCodeAt(0) - "A".charCodeAt(0) + "a".charCodeAt(0));
+        else
+            r += c;
+    }
+    return r;
+}
+function isWordCharOrDollar(c) {
+    return ((c >= "a" && c <= "z") ||
+        (c >= "A" && c <= "Z") ||
+        (c >= "0" && c <= "9") ||
+        c === "_" ||
+        c === "$");
+}
+function isEnumTypeUpper(u) {
+    return u === "ENUM" || u === "ENUM8" || u === "ENUM16";
+}
+class Parser {
+    tokens;
+    pos = 0;
+    hard_error = null;
+    constructor(tokens) {
+        this.tokens = tokens;
+    }
+    run() {
+        /// A lexing error surfaces as a trailing Error token.
+        for (const tok of this.tokens)
+            if (tok.type === TokenType.Error)
+                return Parser.fail(tok.begin, tok.text);
+        const node = this.parseType();
+        if (!node) {
+            if (this.hard_error)
+                return makeResult(null, this.hard_error);
+            return Parser.fail(this.cur().begin, "expected a data type");
+        }
+        if (this.cur().type !== TokenType.End)
+            return Parser.fail(this.cur().begin, "unexpected trailing input after the data type");
+        return makeResult(node, null);
+    }
+    cur() {
+        return this.tokens[this.pos];
+    }
+    type() {
+        return this.tokens[this.pos].type;
+    }
+    advance() {
+        if (this.tokens[this.pos].type !== TokenType.End)
+            ++this.pos;
+    }
+    static fail(at, msg) {
+        return makeResult(null, { message: msg, position: at });
+    }
+    setHardError(at, msg) {
+        if (!this.hard_error)
+            this.hard_error = { message: msg, position: at };
+    }
+    isIdentifier() {
+        return (this.type() === TokenType.Word || this.type() === TokenType.QuotedIdent);
+    }
+    /// Consume consecutive Word tokens iff they all match `words`
+    /// (case-insensitive). On a full match, advances `pos` past them and returns
+    /// true; otherwise leaves `pos` unchanged and returns false.
+    matchWords(words) {
+        let p = this.pos;
+        for (const w of words) {
+            const tok = this.tokens[p];
+            if (tok.type !== TokenType.Word || toUpper(tok.text) !== toUpper(w))
+                return false;
+            ++p;
+        }
+        this.pos = p;
+        return true;
+    }
+    /// Read a single identifier (bare or quoted) into `name`.
+    parseIdentifier() {
+        if (!this.isIdentifier())
+            return { ok: false, name: "" };
+        const name = this.cur().text;
+        this.advance();
+        return { ok: true, name };
+    }
+    parseType() {
+        const id = this.parseIdentifier();
+        if (!id.ok)
+            return null;
+        let type_name = id.name;
+        /// Reject quoted garbage that cannot be a type name (e.g. `x.y`, `Null`).
+        {
+            let allWordChar = true;
+            for (let i = 0; i < type_name.length; ++i) {
+                if (!isWordCharOrDollar(type_name[i])) {
+                    allWordChar = false;
+                    break;
+                }
+            }
+            if (!allWordChar)
+                return null;
+        }
+        const type_name_upper = toUpper(type_name);
+        /// Keywords that the column-declaration parser claims before the type.
+        if (type_name_upper === "NOT" ||
+            type_name_upper === "NULL" ||
+            type_name_upper === "DEFAULT" ||
+            type_name_upper === "MATERIALIZED" ||
+            type_name_upper === "EPHEMERAL" ||
+            type_name_upper === "ALIAS" ||
+            type_name_upper === "AUTO" ||
+            type_name_upper === "PRIMARY" ||
+            type_name_upper === "COMMENT" ||
+            type_name_upper === "CODEC")
+            return null;
+        /// SQL-standard multi-word type names.
+        const suffix = this.parseTypeNameSuffix(type_name_upper);
+        if (suffix !== "")
+            type_name = type_name_upper + " " + suffix;
+        this.skipTrailingComma();
+        /// Enum special case -> EnumDataType with explicit values.
+        if (isEnumTypeUpper(type_name_upper) &&
+            this.type() === TokenType.OpeningParen) {
+            const saved = this.pos;
+            this.advance();
+            const values = [];
+            if (this.parseEnumValues(values) &&
+                this.type() === TokenType.ClosingParen) {
+                this.advance();
+                const node = makeNode(NodeKind.EnumDataType);
+                node.name = type_name;
+                node.values = values;
+                return node;
+            }
+            this.pos = saved;
+        }
+        /// Tuple special case -> TupleDataType with optional element names.
+        if (type_name === "Tuple" && this.type() === TokenType.OpeningParen) {
+            const tuple = this.parseTuple(type_name);
+            if (tuple)
+                return tuple;
+            /// else: fall through to the generic path
+        }
+        const node = makeNode(NodeKind.DataType);
+        node.name = type_name;
+        if (this.type() !== TokenType.OpeningParen)
+            return node;
+        this.advance();
+        if (!this.parseArgumentList(type_name, node.arguments))
+            return null;
+        if (this.type() !== TokenType.ClosingParen)
+            return null;
+        this.advance();
+        node.has_argument_list = true;
+        return node;
+    }
+    /// Returns the suffix to append for SQL-standard multi-word names, or "".
+    parseTypeNameSuffix(u) {
+        if (u === "NATIONAL") {
+            if (this.matchWords(["CHARACTER", "LARGE", "OBJECT"]))
+                return "CHARACTER LARGE OBJECT";
+            if (this.matchWords(["CHARACTER", "VARYING"]))
+                return "CHARACTER VARYING";
+            if (this.matchWords(["CHAR", "VARYING"]))
+                return "CHAR VARYING";
+            if (this.matchWords(["CHARACTER"]))
+                return "CHARACTER";
+            if (this.matchWords(["CHAR"]))
+                return "CHAR";
+        }
+        else if (u === "BINARY" ||
+            u === "CHARACTER" ||
+            u === "CHAR" ||
+            u === "NCHAR") {
+            if (this.matchWords(["LARGE", "OBJECT"]))
+                return "LARGE OBJECT";
+            if (this.matchWords(["VARYING"]))
+                return "VARYING";
+        }
+        else if (u === "DOUBLE") {
+            if (this.matchWords(["PRECISION"]))
+                return "PRECISION";
+        }
+        else if (u.indexOf("INT") !== -1) {
+            /// MySQL-compatible SIGNED / UNSIGNED, optionally after `(width)`.
+            if (this.matchWords(["SIGNED"]))
+                return "SIGNED";
+            if (this.matchWords(["UNSIGNED"]))
+                return "UNSIGNED";
+            if (this.type() === TokenType.OpeningParen) {
+                const saved = this.pos;
+                this.advance();
+                if (this.type() === TokenType.Number)
+                    this.advance();
+                if (this.type() === TokenType.ClosingParen) {
+                    this.advance();
+                    if (this.matchWords(["SIGNED"]))
+                        return "SIGNED";
+                    if (this.matchWords(["UNSIGNED"]))
+                        return "UNSIGNED";
+                }
+                else {
+                    /// not the width form; leave the paren for generic args
+                    this.pos = saved;
+                }
+            }
+        }
+        return "";
+    }
+    /// Skip a trailing comma right before a closing paren: `Tuple(Int, String,)`.
+    skipTrailingComma() {
+        if (this.type() === TokenType.Comma &&
+            this.tokens[this.pos + 1].type === TokenType.ClosingParen)
+            this.advance();
+    }
+    /// Explicit-only enum body: 'name' = value, ... . Returns false (caller
+    /// restores) for auto-assigned or otherwise non-trivial enums.
+    parseEnumValues(values) {
+        let first = true;
+        while (true) {
+            if (!first) {
+                if (this.type() !== TokenType.Comma)
+                    break;
+                this.advance();
+            }
+            first = false;
+            if (this.type() !== TokenType.String)
+                return false;
+            const name = this.cur().text;
+            this.advance();
+            if (this.type() !== TokenType.Equals)
+                return false;
+            this.advance();
+            let negative = false;
+            if (this.type() === TokenType.Minus) {
+                negative = true;
+                this.advance();
+            }
+            if (this.type() !== TokenType.Number || this.cur().is_float)
+                return false;
+            const v = BigInt(this.cur().text);
+            this.advance();
+            values.push({ name, value: negative ? -v : v });
+        }
+        return values.length !== 0;
+    }
+    /// Parse a Tuple body into element types + names. Returns null (with the
+    /// position restored) if it cannot, so the caller can try the generic path.
+    parseTuple(type_name) {
+        const saved = this.pos;
+        this.advance(); /// consume '('
+        const node = makeNode(NodeKind.TupleDataType);
+        node.name = type_name;
+        const names = [];
+        let has_named = false;
+        let first = true;
+        while (true) {
+            if (!first) {
+                if (this.type() === TokenType.Comma)
+                    this.advance();
+                else
+                    break;
+            }
+            first = false;
+            const element_pos = this.pos;
+            /// Try: identifier Type  (named element)
+            const id = this.parseIdentifier();
+            if (id.ok) {
+                const t = this.parseType();
+                if (t) {
+                    names.push(id.name);
+                    node.arguments.push(t);
+                    has_named = true;
+                    continue;
+                }
+            }
+            /// Else: just Type  (unnamed element)
+            this.pos = element_pos;
+            const t = this.parseType();
+            if (t) {
+                names.push("");
+                node.arguments.push(t);
+            }
+            else {
+                break;
+            }
+        }
+        if (this.type() === TokenType.ClosingParen && node.arguments.length !== 0) {
+            this.advance();
+            node.has_argument_list = true;
+            if (has_named)
+                node.element_names = names;
+            return node;
+        }
+        this.pos = saved;
+        return null;
+    }
+    /// The generic comma-separated argument list inside `Type(...)`.
+    parseArgumentList(type_name, out) {
+        const lower = toLower(type_name);
+        if (type_name === "AggregateFunction" ||
+            type_name === "SimpleAggregateFunction") {
+            this.setHardError(this.cur().begin, type_name + " is not supported by this parser yet");
+            return false;
+        }
+        if (lower === "json") {
+            this.setHardError(this.cur().begin, "JSON typed/object arguments are not supported by this parser yet");
+            return false;
+        }
+        let arg_num = 0;
+        while (true) {
+            if (arg_num > 0) {
+                if (this.type() === TokenType.Comma)
+                    this.advance();
+                else
+                    break;
+            }
+            let arg;
+            if (type_name === "Dynamic")
+                arg = this.parseEqualsArgument();
+            else if (type_name === "Nested")
+                arg = this.parseNameTypePair();
+            else if (type_name === "Tuple")
+                arg = this.parseNameTypePairOrType();
+            else
+                arg = this.parseGenericArgument();
+            if (!arg)
+                break;
+            out.push(arg);
+            ++arg_num;
+        }
+        return true;
+    }
+    /// `identifier = number` -> Function equals(Identifier, Literal).
+    parseEqualsArgument() {
+        const id = this.parseIdentifier();
+        if (!id.ok)
+            return null;
+        if (this.type() !== TokenType.Equals)
+            return null;
+        this.advance();
+        const number = this.parseNumberLiteral();
+        if (!number)
+            return null;
+        const idNode = makeNode(NodeKind.Identifier);
+        idNode.name = id.name;
+        const fn = makeNode(NodeKind.Function);
+        fn.name = "equals";
+        fn.is_operator = true;
+        fn.arguments = [idNode, number];
+        return fn;
+    }
+    /// `name Type` -> NameTypePair (Nested elements).
+    parseNameTypePair() {
+        const id = this.parseIdentifier();
+        if (!id.ok)
+            return null;
+        const t = this.parseType();
+        if (!t)
+            return null;
+        const node = makeNode(NodeKind.NameTypePair);
+        node.name = id.name;
+        node.data_type = t;
+        return node;
+    }
+    parseNameTypePairOrType() {
+        const saved = this.pos;
+        const pair = this.parseNameTypePair();
+        if (pair)
+            return pair;
+        this.pos = saved;
+        return this.parseType();
+    }
+    /// Generic argument: a scalar literal (optionally `lit = lit`), or a type.
+    parseGenericArgument() {
+        const lit = this.parseScalarLiteral();
+        if (lit) {
+            if (this.type() === TokenType.Equals) {
+                this.advance();
+                const rhs = this.parseScalarLiteral();
+                if (!rhs)
+                    return null;
+                const fn = makeNode(NodeKind.Function);
+                fn.name = "equals";
+                fn.is_operator = true;
+                fn.arguments = [lit, rhs];
+                return fn;
+            }
+            return lit;
+        }
+        return this.parseType();
+    }
+    parseNumberLiteral() {
+        let negative = false;
+        if (this.type() === TokenType.Minus) {
+            negative = true;
+            this.advance();
+        }
+        if (this.type() !== TokenType.Number)
+            return null;
+        const node = makeNode(NodeKind.Literal);
+        node.value_type = this.cur().is_float
+            ? "Float64"
+            : negative
+                ? "Int64"
+                : "UInt64";
+        node.value = (negative ? "-" : "") + this.cur().text;
+        this.advance();
+        return node;
+    }
+    /// A scalar literal: number (optionally signed) or string.
+    parseScalarLiteral() {
+        if (this.type() === TokenType.Number || this.type() === TokenType.Minus)
+            return this.parseNumberLiteral();
+        if (this.type() === TokenType.String) {
+            const node = makeNode(NodeKind.Literal);
+            node.value_type = "String";
+            node.value = this.cur().text;
+            this.advance();
+            return node;
+        }
+        return null;
+    }
+}
+function makeResult(ast, error) {
+    return {
+        ast,
+        error,
+        ok() {
+            return this.ast !== null;
+        },
+    };
+}
+/// Parse the whole string as a single data type. Trailing tokens after a
+/// complete type are an error (the entire input must be one type).
+export function parseDataType(input) {
+    const parser = new Parser(tokenize(input));
+    return parser.run();
+}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@clickhouse/datatype-parser",
+  "version": "0.1.0",
+  "description": "Standalone ClickHouse data-type string parser — a TypeScript port of the chdt C++ library.",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ClickHouse/clickhouse-js.git",
+    "directory": "packages/datatype-parser"
+  },
+  "keywords": [
+    "clickhouse",
+    "data-type",
+    "parser",
+    "ast",
+    "rowbinary"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "lint": "eslint --max-warnings=0 .",
+    "lint:fix": "eslint . --fix",
+    "prepack": "npm run build",
+    "parse": "node tool/main.ts",
+    "test": "node --import tsx --test test/*.test.ts",
+    "test:oracle": "tsx test/oracle_compare.ts",
+    "test:unsupported": "tsx test/check_unsupported.ts",
+    "snapshot:update": "tsx test/update_snapshots.ts",
+    "validate:live": "tsx test/validate_types_live.ts"
+  },
+  "devDependencies": {
+    "tsx": "^4.22.4",
+    "typescript": "^5.6.0"
+  }
+}