@xdbml/parse 0.1.0-poc.1

package/dist/keywords.js

@@ -0,0 +1,277 @@
+/**
+ * Shared keyword vocabulary for xDBML tokenizers and highlighters.
+ *
+ * This module is the single source of truth for the keyword lists
+ * that drive syntax highlighting across multiple surfaces:
+ *
+ *   - parser/src/monarch.ts        the playground's in-editor highlighter
+ *   - tools/textmate/...           the TextMate grammar for VS Code,
+ *                                  Shiki (xdbml.org code blocks, Claude
+ *                                  chat code blocks, etc.), and GitHub
+ *
+ * The grammar in grammar/xDBML.g4 is the language's canonical
+ * specification; this file mirrors its keyword vocabulary in a form
+ * convenient for consumers. A keyword-consistency test (in
+ * parser/test/) asserts that every keyword listed here is recognized
+ * by the parser.
+ *
+ * When adding a keyword to xDBML:
+ *   1. Update the grammar in grammar/xDBML.g4
+ *   2. Update parser/src/parser.ts if the parser needs to recognize it
+ *   3. Add it to the right array below
+ *   4. Re-run the TextMate grammar build script
+ *      (tools/textmate/scripts/build.mjs)
+ *   5. Run `npm test` in the parser package to verify all three
+ *      consumers see the keyword
+ *
+ * All keywords here are case-insensitive in xDBML. Stored in
+ * lower-case as the canonical form; matchers should be case-insensitive.
+ */
+/* -------------------------------------------------------------------------
+ * Declaration keywords
+ *
+ * The keywords that introduce a top-level declaration. The full set
+ * includes both container-style keywords (Container, Schema, etc.)
+ * and entity-style keywords (Table, Entity, etc.) plus other
+ * top-level constructs (Ref, View, Note, Enum, etc.).
+ * ----------------------------------------------------------------------- */
+export const CONTAINER_KEYWORDS = [
+    'container',
+    'schema',
+    'database',
+    'keyspace',
+    'namespace',
+    'dataset',
+    'bucket',
+];
+export const ENTITY_KEYWORDS = [
+    'table',
+    'entity',
+    'collection',
+    'record',
+];
+/**
+ * The full set of declaration keywords. Includes containers,
+ * entities, and other top-level constructs.
+ */
+export const DECLARATION_KEYWORDS = [
+    'project',
+    ...CONTAINER_KEYWORDS,
+    ...ENTITY_KEYWORDS,
+    'type',
+    'edge',
+    'view',
+    'enum',
+    'ref',
+    'note',
+    'tablepartial',
+    'tablegroup',
+    'diagramview',
+];
+/* -------------------------------------------------------------------------
+ * Type expression keywords
+ *
+ * Keywords that appear inside a field's type expression, after the
+ * colon. Includes structural types (object, array, map) and
+ * polymorphism markers (oneOf, anyOf, allOf, union).
+ * ----------------------------------------------------------------------- */
+export const STRUCTURAL_TYPE_KEYWORDS = [
+    'object',
+    'struct',
+    'array',
+    'list',
+    'map',
+    'dict',
+    'dictionary',
+    'set',
+    'json',
+    'jsonb',
+    'variant',
+];
+export const POLYMORPHISM_KEYWORDS = [
+    'union',
+    'oneof',
+    'anyof',
+    'allof',
+];
+/* -------------------------------------------------------------------------
+ * Scalar and BSON types
+ *
+ * The parser accepts any identifier as a scalar type (open vocabulary),
+ * so these lists drive color, not validation. Themes color them under
+ * `storage.type` (TextMate) or `type` (Monarch).
+ * ----------------------------------------------------------------------- */
+export const SCALAR_TYPES = [
+    // Integers
+    'tinyint', 'smallint', 'mediumint', 'int', 'integer', 'bigint',
+    'int32', 'int64',
+    // Floating point and decimal
+    'float', 'double', 'decimal', 'dec', 'numeric', 'real',
+    // Boolean
+    'bit', 'bool', 'boolean',
+    // Strings
+    'char', 'varchar', 'varchar2', 'nvarchar', 'nvarchar2', 'nchar',
+    'text', 'mediumtext', 'longtext', 'string', 'ntext',
+    // Binary
+    'binary', 'varbinary', 'blob', 'mediumblob', 'longblob', 'tinyblob',
+    'tinytext',
+    // Document
+    'json', 'jsonb', 'variant', 'xml',
+    // Date/time
+    'date', 'time', 'datetime', 'datetime2', 'timestamp',
+    'timestamptz', 'year',
+    // Identity and network
+    'uuid', 'inet6',
+    // Money
+    'money', 'smallmoney',
+    // Enumeration as a type position
+    'enum',
+];
+export const BSON_TYPES = [
+    'objectid',
+    'decimal128',
+    'bindata',
+    'minkey',
+    'maxkey',
+    'symbol',
+    'regex',
+    'long',
+    'double',
+];
+/* -------------------------------------------------------------------------
+ * Settings vocabulary
+ *
+ * Settings appear inside `[...]` brackets after a field declaration
+ * or other construct. Two forms:
+ *
+ *   - Flag settings: bare names like `pk`, `unique`, `not null`
+ *   - Keyed settings: `name: value` pairs
+ *
+ * `required` is normalized to `not null` by the parser per spec §8;
+ * both spellings get the same color.
+ *
+ * Setting keys are open-vocabulary at the parser level (any identifier
+ * followed by `:` is accepted as a setting), so this list drives
+ * highlighting only.
+ * ----------------------------------------------------------------------- */
+export const SETTING_FLAGS = [
+    'pk',
+    'primary',
+    'key',
+    'unique',
+    'null',
+    'not',
+    'required',
+    'increment',
+    'inactive', // v0.2 §11.9: Ref flag for visualization-only deactivation
+];
+export const SETTING_KEYS = [
+    // General
+    'note',
+    'default',
+    'ref',
+    'name',
+    'color',
+    'headercolor',
+    'as',
+    'check',
+    // xDBML-specific
+    'type',
+    'target',
+    'targets',
+    'database_type',
+    'source',
+    'source_cardinality',
+    'target_cardinality',
+    'min_source',
+    'max_source',
+    'min_target',
+    'max_target',
+    'undirected',
+    'discriminator',
+    'source_query',
+    'materialized',
+    'refresh_schedule',
+    'refresh_on',
+    'source_database',
+    'storage_options',
+    // Validation
+    'pattern',
+    'format',
+    'minlength',
+    'maxlength',
+    'minimum',
+    'maximum',
+    'exclusiveminimum',
+    'exclusivemaximum',
+    'multipleof',
+    'minitems',
+    'maxitems',
+    'uniqueitems',
+    'minproperties',
+    'maxproperties',
+    // AI-readiness
+    'synonyms',
+    'business_term',
+    'granularity',
+    'tags',
+    // Referential actions
+    'delete',
+    'update',
+    // Block keywords (entity-body or top-level)
+    'indexes',
+    'checks', // v0.2 §10: entity-level checks block
+    // Container settings
+    'replication',
+    'location',
+    'default_charset',
+    // Module system (v0.2 §26)
+    'cloned_at', // v0.2 directive setting: ISO 8601 timestamp on a use/reuse directive
+];
+/* -------------------------------------------------------------------------
+ * Value vocabularies
+ *
+ * Constants that appear as right-hand-side values for specific
+ * settings (granularity for time-series, etc.). The parser doesn't
+ * validate these; they're highlighted as constants when recognized.
+ * ----------------------------------------------------------------------- */
+export const GRANULARITY_VALUES = [
+    'year', 'quarter', 'month', 'week', 'day',
+    'hour', 'minute', 'second',
+    'millisecond', 'microsecond', 'nanosecond',
+];
+/* -------------------------------------------------------------------------
+ * Top-level directives
+ *
+ * Appear only at the very top of a file: `xdbml: 0.1` or
+ * `experimental: ...`. Distinct from declaration keywords because
+ * their syntax (and meaning) is different.
+ * ----------------------------------------------------------------------- */
+export const DIRECTIVE_KEYWORDS = [
+    'xdbml',
+    'experimental',
+];
+/* -------------------------------------------------------------------------
+ * Module-system keywords  (v0.2, spec §26)
+ *
+ * The keywords that introduce a module-system directive:
+ *
+ *     reuse { entity X, type Y as Z } from './path' [cloned_at: '...'] { ... }
+ *     use * from './path'
+ *
+ * `use` and `reuse` start the directive; `from` precedes the path
+ * literal; `as` appears inside the import-item list when renaming.
+ *
+ * `as` is also a contextual keyword in upstream DBML (`Table users as u`
+ * for aliasing), so it's already implicitly highlighted via its
+ * presence in SETTING_KEYS; including it here as well doesn't change
+ * behavior but documents the module-system role.
+ *
+ * Highlighted with the scope `keyword.control.module.xdbml`.
+ * ----------------------------------------------------------------------- */
+export const MODULE_KEYWORDS = [
+    'use',
+    'reuse',
+    'from',
+    'as',
+];

package/dist/lexer.d.ts

@@ -0,0 +1,91 @@
+/**
+ * xDBML lexer.
+ *
+ * Produces a stream of tokens with line/column positions. Keywords are NOT
+ * recognized as distinct token kinds at the lexer level. Identifiers carry
+ * their source value, and the parser interprets them as keywords by
+ * lowercased string comparison (per spec §3.8: keywords are
+ * case-insensitive, identifiers are case-sensitive — both end up as
+ * IDENTIFIER tokens here, with the parser making the keyword decision).
+ *
+ * Special multi-character punctuation handled here rather than in the parser:
+ *   - `<>` is lexed as MANY_TO_MANY (otherwise the parser would see `<` then `>`)
+ *   - `[*]` is lexed as ARRAY_WILDCARD only when followed-by/preceded-by a dot
+ *     in a path context; otherwise it's left as `LBRACKET`, `OP(*)`, `RBRACKET`
+ *     to avoid ambiguity with array settings. In practice the parser only
+ *     uses ARRAY_WILDCARD inside fieldPath, so we lex `[*]` greedily whenever
+ *     we see it and let the parser decide based on context.
+ *   - `'''...'''` lexes as a single STRING_LITERAL with the multiline flag.
+ */
+import type { Position } from './ast.ts';
+export declare const TokenKind: {
+    readonly Identifier: "Identifier";
+    readonly QuotedIdentifier: "QuotedIdentifier";
+    readonly StringLiteral: "StringLiteral";
+    readonly MultilineString: "MultilineString";
+    readonly NumberLiteral: "NumberLiteral";
+    readonly ExpressionLiteral: "ExpressionLiteral";
+    readonly LBrace: "LBrace";
+    readonly RBrace: "RBrace";
+    readonly LBracket: "LBracket";
+    readonly RBracket: "RBracket";
+    readonly LParen: "LParen";
+    readonly RParen: "RParen";
+    readonly Comma: "Comma";
+    readonly Colon: "Colon";
+    readonly Dot: "Dot";
+    readonly Tilde: "Tilde";
+    readonly Semicolon: "Semicolon";
+    readonly LAngle: "LAngle";
+    readonly RAngle: "RAngle";
+    readonly Minus: "Minus";
+    readonly ManyToMany: "ManyToMany";
+    readonly ArrayWildcard: "ArrayWildcard";
+    /**
+     * Standalone `*`. Used by v0.2 module-system import-all directives:
+     * `use * from './path'`. Note that `[*]` is a separate token
+     * (ArrayWildcard); this Star token is only produced when the asterisk
+     * appears outside that context.
+     */
+    readonly Star: "Star";
+    readonly EOF: "EOF";
+};
+export type TokenKind = typeof TokenKind[keyof typeof TokenKind];
+export interface Token {
+    kind: TokenKind;
+    /** The raw source text of the token */
+    text: string;
+    /** For StringLiteral / MultilineString / QuotedIdentifier: text with quotes stripped & escapes processed */
+    value?: string;
+    start: Position;
+    end: Position;
+}
+export declare class LexError extends Error {
+    position: Position;
+    constructor(message: string, position: Position);
+}
+export declare class Lexer {
+    private text;
+    private offset;
+    private line;
+    private column;
+    constructor(text: string);
+    private pos;
+    private peek;
+    private advance;
+    private matchSeq;
+    private isAtEnd;
+    private skipTrivia;
+    private isIdentStart;
+    private isIdentCont;
+    private isDigit;
+    private lexIdentifier;
+    private lexNumber;
+    private lexString;
+    private lexQuotedIdent;
+    private lexBacktick;
+    /** Read a single token. Returns EOF when out of input. */
+    private nextToken;
+    tokenize(): Token[];
+}
+export declare function tokenize(text: string): Token[];