@clickhouse/datatype-parser 0.1.0 → 0.1.2

package/README.md

@@ -1,4 +1,4 @@
-# chdt-ts — standalone ClickHouse data-type parser (TypeScript)
+# @clickhouse/datatype-parser
 
 A small, self-contained TypeScript library that parses a ClickHouse **data-type
 string** (the kind sent in the types row of `RowBinaryWithNamesAndTypes`, e.g.
@@ -13,30 +13,10 @@ frozen `EXPLAIN AST json = 1` document (format **version 2**), so its output is
 drop-in match for what the server produces — and is **byte-identical** to the C++
 parser's output across the full test corpus.
 
-## Layout
-
-The module structure tracks the C++ sources one-to-one:
-
-| TypeScript      | ported from (C++)             | role                                    |
-| --------------- | ----------------------------- | --------------------------------------- |
-| `src/ast.ts`    | `include/chdt/ast.h`          | the AST node shape + `makeNode` factory |
-| `src/lexer.ts`  | `src/lexer.{h,cpp}`           | the purpose-built tokenizer             |
-| `src/parser.ts` | `src/parser.cpp` + `parser.h` | the `ParserDataType::parseImpl` port    |
-| `src/json.ts`   | `src/json.cpp`                | the byte-faithful JSON serializer       |
-| `src/index.ts`  | —                             | public barrel                           |
-| `tool/main.ts`  | `tool/main.cpp`               | the `chdt-parse` CLI                    |
-
-The lexer and parser deliberately preserve the original control flow, branch
-ordering, helper names, and `pos` save/restore points. A few signatures changed
-where C++ used out-parameters (`std::string &`): `parseIdentifier` and
-`decodeQuoted` return small result objects instead.
-
 ## Install & build
 
 ```bash
-npm install
-npm run build      # emits dist/ (JS + .d.ts)
-npm run typecheck  # tsc --noEmit
+npm install @clickhouse/datatype-parser
 ```
 
 ## Usage
@@ -55,11 +35,291 @@ if (r.ok()) {
 }
 ```
 
-CLI (no build step needed — runs on Node.js 24+ native TypeScript support):
+## AST
+
+```sql
+Tuple(
+  id UInt64,
+  name LowCardinality(String),
+  price Decimal(18, 4),
+  ts DateTime64(9, 'UTC'),
+  tags Array(LowCardinality(Nullable(String))),
+  attrs Map(String, Array(Nullable(Int32))),
+  status Enum8('active' = 1, 'closed' = -2),
+  coords Array(Tuple(Float64, Float64)),
+  meta Nested(k String, v UInt32),
+  fixed FixedString(16),
+  dyn Dynamic(max_types = 8),
+  variant Variant(UInt64, String, Array(UInt8)),
+  raw Object('json')
+)
+```
+
+```json
+{
+  "type": "TupleDataType",
+  "name": "Tuple",
+  "arguments": [
+    {
+      "type": "DataType",
+      "name": "UInt64"
+    },
+    {
+      "type": "DataType",
+      "name": "LowCardinality",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "String"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Decimal",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "18"
+        },
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "4"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "DateTime64",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "9"
+        },
+        {
+          "type": "Literal",
+          "value_type": "String",
+          "value": "UTC"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Array",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "LowCardinality",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "Nullable",
+              "arguments": [
+                {
+                  "type": "DataType",
+                  "name": "String"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Map",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "String"
+        },
+        {
+          "type": "DataType",
+          "name": "Array",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "Nullable",
+              "arguments": [
+                {
+                  "type": "DataType",
+                  "name": "Int32"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "EnumDataType",
+      "name": "Enum8",
+      "values": [
+        {
+          "name": "active",
+          "value": 1
+        },
+        {
+          "name": "closed",
+          "value": -2
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Array",
+      "arguments": [
+        {
+          "type": "TupleDataType",
+          "name": "Tuple",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "Float64"
+            },
+            {
+              "type": "DataType",
+              "name": "Float64"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Nested",
+      "arguments": [
+        {
+          "type": "NameTypePair",
+          "name": "k",
+          "data_type": {
+            "type": "DataType",
+            "name": "String"
+          }
+        },
+        {
+          "type": "NameTypePair",
+          "name": "v",
+          "data_type": {
+            "type": "DataType",
+            "name": "UInt32"
+          }
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "FixedString",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "16"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Dynamic",
+      "arguments": [
+        {
+          "type": "Function",
+          "name": "equals",
+          "is_operator": true,
+          "arguments": [
+            {
+              "type": "Identifier",
+              "name": "max_types"
+            },
+            {
+              "type": "Literal",
+              "value_type": "UInt64",
+              "value": "8"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Variant",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "UInt64"
+        },
+        {
+          "type": "DataType",
+          "name": "String"
+        },
+        {
+          "type": "DataType",
+          "name": "Array",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "UInt8"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Object",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "String",
+          "value": "json"
+        }
+      ]
+    }
+  ],
+  "element_names": [
+    "id",
+    "name",
+    "price",
+    "ts",
+    "tags",
+    "attrs",
+    "status",
+    "coords",
+    "meta",
+    "fixed",
+    "dyn",
+    "variant",
+    "raw"
+  ]
+}
+```
+
+## Layout
+
+The module structure tracks the C++ sources one-to-one:
+
+| TypeScript      | ported from (C++)             | role                                    |
+| --------------- | ----------------------------- | --------------------------------------- |
+| `src/ast.ts`    | `include/chdt/ast.h`          | the AST node shape + `makeNode` factory |
+| `src/lexer.ts`  | `src/lexer.{h,cpp}`           | the purpose-built tokenizer             |
+| `src/parser.ts` | `src/parser.cpp` + `parser.h` | the `ParserDataType::parseImpl` port    |
+| `src/json.ts`   | `src/json.cpp`                | the byte-faithful JSON serializer       |
+| `src/index.ts`  | —                             | public barrel                           |
+
+The lexer and parser deliberately preserve the original control flow, branch
+ordering, helper names, and `pos` save/restore points. A few signatures changed
+where C++ used out-parameters (`std::string &`): `parseIdentifier` and
+`decodeQuoted` return small result objects instead.
+
+## Development
 
 ```bash
-npm run parse -- "Array(Nullable(UInt64))"
-echo "Enum8('a' = 1, 'b' = 2)" | npm run parse
+npm install
+npm run build      # emits dist/ (JS + .d.ts)
+npm run typecheck  # tsc --noEmit
 ```
 
 ## Output shape

package/dist/parser.js

@@ -40,6 +40,17 @@ function isWordCharOrDollar(c) {
 function isEnumTypeUpper(u) {
     return u === "ENUM" || u === "ENUM8" || u === "ENUM16";
 }
+/// Does `text` parse in FULL as a numeric literal — the TS equivalent of the
+/// C++ `strtod` (float) / `strtoull` (integer) + end-pointer check? The lexer
+/// is deliberately lenient and can hand back a half-formed lexeme such as `1e`
+/// (an exponent with no digits); those must be rejected. Integer lexemes the
+/// lexer emits are pure digit runs (so always valid); floats may carry a
+/// malformed exponent, so the optional exponent here requires >= 1 digit.
+function isFullyParsedNumber(text, isFloat) {
+    if (isFloat)
+        return /^(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/.test(text);
+    return /^\d+$/.test(text);
+}
 class Parser {
     tokens;
     pos = 0;
@@ -419,13 +430,26 @@ class Parser {
         }
         if (this.type() !== TokenType.Number)
             return null;
+        /// The lexer only SKIPS over a number without checking correctness (the
+        /// server's `Lexer` does the same: "not to parse a number or check
+        /// correctness, but only to skip it"). The server then rejects a malformed
+        /// literal when it converts the token text into a `Field`; this port has no
+        /// such stage, so validate here by parsing the lexeme in full. Without it a
+        /// bare exponent like `1e` would be emitted verbatim into a `Float64`
+        /// literal's JSON `value`, yielding invalid JSON (`"value":1e`). Mirrors the
+        /// C++ `strtod` / `strtoull` + end-pointer check in `parser.cpp`.
+        const text = this.cur().text;
+        if (!isFullyParsedNumber(text, this.cur().is_float)) {
+            this.setHardError(this.cur().begin, `malformed numeric literal: '${text}'`);
+            return null;
+        }
         const node = makeNode(NodeKind.Literal);
         node.value_type = this.cur().is_float
             ? "Float64"
             : negative
                 ? "Int64"
                 : "UInt64";
-        node.value = (negative ? "-" : "") + this.cur().text;
+        node.value = (negative ? "-" : "") + text;
         this.advance();
         return node;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clickhouse/datatype-parser",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Standalone ClickHouse data-type string parser — a TypeScript port of the chdt C++ library.",
   "license": "Apache-2.0",
   "type": "module",
@@ -41,9 +41,11 @@
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "lint": "eslint --max-warnings=0 .",
     "lint:fix": "eslint . --fix",
+    "pack": "npm pack",
     "prepack": "npm run build",
-    "parse": "node tool/main.ts",
-    "test": "node --import tsx --test test/*.test.ts",
+    "bench": "vitest bench --run",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "test:oracle": "tsx test/oracle_compare.ts",
     "test:unsupported": "tsx test/check_unsupported.ts",
     "snapshot:update": "tsx test/update_snapshots.ts",
@@ -51,6 +53,7 @@
   },
   "devDependencies": {
     "tsx": "^4.22.4",
-    "typescript": "^5.6.0"
+    "typescript": "^5.6.0",
+    "vitest": "^4.0.16"
   }
 }