@willbooster/tree-sitter-haskell 0.0.0-stage → 1.0.0

package/grammar/exp.js

@@ -0,0 +1,413 @@
+const {
+  sep1,
+  sep,
+  parens,
+  braces,
+  brackets,
+  layout_sort,
+  layout,
+  unboxed_tuple_nonempty,
+  unboxed_sum_single,
+  qualified,
+} = require('./util.js')
+
+module.exports = {
+
+  // ------------------------------------------------------------------------
+  // names
+  // ------------------------------------------------------------------------
+
+  _exp_name: $ => choice(
+    $._cons,
+    $._vars,
+    $.variable,
+    $.implicit_variable,
+    $.label,
+  ),
+
+  _exp_th_quoted_name: $ => choice(
+    seq('\'', field('name', choice($._vars, $._cons))),
+    prec('prefix', seq('\'\'', field('type', $.type))),
+  ),
+
+  // ------------------------------------------------------------------------
+  // tuples and parens
+  // ------------------------------------------------------------------------
+
+  _exp_parens: $ => parens($, field('expression', $._exp)),
+
+  // Having this separate reduces size by ~15kB
+  _exp_tuple_elems: $ => seq(
+    choice(
+      seq(repeat1(','), field('element', $._exp)),
+      seq(field('element', $._exp), ',', optional(field('element', $._exp))),
+    ),
+    repeat(seq(',', optional(field('element', $._exp))))
+  ),
+
+  /**
+    * This needs to be disambiguated from `prefix_tuple`, which is a constructor with _only_ commas.
+    * Tuple sections aren't allowed in patterns.
+    *
+    * Since tuple expressions can contain singular expressions in sections like `(a,)` and `(,a)`, it has to be ensured
+    * that there is _at least_ each one comma and one expression in there, but the comma may be on either side and be
+    * preceded by any number of further commas, like `(,,,a)`.
+    *
+    * The final `repeat` is simpler, it just has to ensure that no two `_exp`s can be successive, but this encoding
+    * means that the optional `_exp` after `(5,)` needs to be included in the `choice`, otherwise a simple pair
+    * would be impossible.
+    */
+  _exp_tuple: $ => parens($, $._exp_tuple_elems),
+
+  /**
+   * Unlike their boxed variants, unboxed tuples may be nullary and unary, making it simpler to parse them.
+   */
+  _exp_unboxed_tuple: $ => unboxed_tuple_nonempty($, $._exp),
+
+  /**
+   * Unboxed sums must have at least one separating `|`, otherwise the expression would be a unary or nullary tuple.
+   * This is a lenient parser that allows multiple variants to be filled in, for simplicity.
+   */
+  _exp_unboxed_sum: $ => unboxed_sum_single($, $._exp),
+
+  // ------------------------------------------------------------------------
+  // lists
+  // ------------------------------------------------------------------------
+
+  _exp_list: $ => brackets($, sep1(',', field('element', $._exp)), optional(',')),
+
+  /**
+    * An expression like `[1,2..20]`.
+    */
+  _exp_arithmetic_sequence: $ => brackets(
+    $,
+    field('from', $._exp),
+    optional(seq(',', field('step', $._exp))),
+    $._dotdot,
+    optional(field('to', $._exp)),
+  ),
+
+  /**
+   * `TransformListComp`, group style.
+   *
+   * Inlining `_exp` here is necessary to avoid a conflict.
+   */
+  group: $ => seq(
+    'then',
+    'group',
+    optional(seq('by', field('key', choice(alias($._exp_signature, $.signature), $.expression)))),
+    'using',
+    field('classifier', $._exp),
+  ),
+
+  /**
+   * `TransformListComp`, simple transformation style.
+   */
+  transform: $ => seq(
+    'then',
+    field('transformation', $._exp),
+    optional(seq('by', field('key', $._exp))),
+  ),
+
+  qualifier: $ => choice(
+    $.generator,
+    $.let,
+    $.transform,
+    $.group,
+    alias($._exp, $.boolean),
+  ),
+
+  /**
+   * This is a supertype.
+   */
+  qualifiers: $ => seq(sep1(',', field('qualifier', $.qualifier))),
+
+  _exp_list_comprehension: $ => brackets(
+    $,
+    field('expression', $._exp),
+    repeat1(seq('|', field('qualifiers', $.qualifiers))),
+  ),
+
+  // ------------------------------------------------------------------------
+  // greedy block args
+  // ------------------------------------------------------------------------
+
+  _exp_lambda: $ => seq(
+    '\\',
+    field('patterns', $.patterns),
+    $._arrow,
+    field('expression', $._exp),
+  ),
+
+  _exp_let_in: $ => seq($._let, optional($._phantom_in), 'in', field('expression', $._exp)),
+
+  _exp_conditional: $ => seq(
+    'if',
+    field('if', $._exp),
+    repeat(';'),
+    'then',
+    field('then', $._exp),
+    repeat(';'),
+    'else',
+    field('else', $._exp),
+  ),
+
+  /**
+   * These block arguments don't end in a layout, so they all range over all following block arguments and will
+   * therefore always be the last argument in an application or infix chain.
+   * They also pull a trailing type annotation into their body.
+   */
+  _exp_greedy: $ => choice(
+    alias($._exp_lambda, $.lambda),
+    alias($._exp_let_in, $.let_in),
+    alias($._exp_conditional, $.conditional),
+  ),
+
+  // ------------------------------------------------------------------------
+  // do
+  // ------------------------------------------------------------------------
+
+  _exp_statement: $ => $._exp,
+
+  /**
+   * This is a supertype.
+   */
+  statement: $ => choice(
+    alias($._exp_statement, $.exp),
+    alias($.generator, $.bind),
+    $.let,
+    $.rec,
+  ),
+
+  _statements: $ => layout_sort($, $._cmd_layout_start_do, field('statement', $.statement)),
+
+  rec: $ => seq('rec', $._statements),
+
+  _do_keyword: _ => choice('mdo', 'do'),
+
+  do_module: $ => field('qualified_do', qualified($, $._do_keyword)),
+
+  _do: $ => choice(
+    $.do_module,
+    $._do_keyword
+  ),
+
+  _exp_do: $ => seq($._do, $._statements),
+
+  // ------------------------------------------------------------------------
+  // case
+  // ------------------------------------------------------------------------
+
+  match: $ => seq(
+    $._guards,
+    optional($._phantom_arrow),
+    $._arrow,
+    $._cmd_texp_end,
+    field('expression', $._exp),
+  ),
+
+  _simple_match: $ => seq($._arrow, field('expression', $._exp)),
+
+  _matches: $ => field('match', choice(
+    alias($._simple_match, $.match),
+    repeat1($.match),
+  )),
+
+  alternative: $ => seq(
+    field('pattern', $._pat),
+    $._matches,
+    optional($._where_binds),
+  ),
+
+  _nalt: $ => seq(
+    field('patterns', $.patterns),
+    $._matches,
+    optional($._where_binds),
+  ),
+
+  alternatives: $ => layout_sort($, $._cmd_layout_start_case, field('alternative', $.alternative)),
+  _nalts: $ => layout_sort($, $._cmd_layout_start_case, field('alternative', alias($._nalt, $.alternative))),
+
+  _exp_case: $ => seq('case', $._exp, 'of', optional(field('alternatives', $.alternatives))),
+
+  _exp_lambda_case: $ => seq(
+    '\\',
+    'case',
+    optional(field('alternatives', $.alternatives)),
+  ),
+
+  /**
+   * alternatives are not optional in a `\cases` expression, but we're lenient.
+   */
+  _exp_lambda_cases: $ => seq(
+    '\\',
+    'cases',
+    optional(field('alternatives', alias($._nalts, $.alternatives))),
+  ),
+
+  _exp_multi_way_if: $ => seq(
+    'if',
+    $._cmd_layout_start_if,
+    repeat(field('match', $.match)),
+    $._cond_layout_end,
+  ),
+
+  // ------------------------------------------------------------------------
+  // record
+  // ------------------------------------------------------------------------
+
+  field_update: $ => choice(
+    alias('..', $.wildcard),
+    seq(
+      field('field', $._field_spec),
+      optional(seq('=', field('expression', $._exp)))
+    ),
+  ),
+
+  _exp_record: $ => prec('record', seq(
+    field('expression', $.expression),
+    braces($, sep(',', field('field', $.field_update)))),
+  ),
+
+  _exp_projection_selector: $ => parens(
+    $,
+    $._any_prefix_dot,
+    field('field', $.variable),
+    repeat(seq($._tight_dot, field('field', $.variable))),
+  ),
+
+  /**
+   * A dot-syntax field projection like `var.name.othername`.
+   */
+  _exp_projection: $ => seq(
+    prec('projection', seq(
+      field('expression', $.expression),
+      $._tight_dot,
+    )),
+    field('field', $.field_name),
+  ),
+
+  // ------------------------------------------------------------------------
+  // application
+  // ------------------------------------------------------------------------
+
+  explicit_type: $ => parens($, 'type', field('type', $.type)),
+
+  _exp_apply: $ => prec.left('apply', seq(
+    field('function', $.expression),
+    field('argument', choice(
+      $.expression,
+      alias($._at_type, $.type_application),
+      $.explicit_type,
+    )),
+  )),
+
+  // ------------------------------------------------------------------------
+  // operators
+  // ------------------------------------------------------------------------
+
+  _exp_op: $ => choice(
+    $._sym,
+    $._op_ticked,
+    alias($._prefix_dot, $.operator),
+  ),
+
+  _exp_section_left: $ => parens(
+    $,
+    field('left_operand', $.expression),
+    $._cond_left_section_op,
+    field('operator', choice(
+      $._exp_op,
+      $._operator_minus,
+      $._qsym,
+    )),
+  ),
+
+  _exp_section_right: $ => parens(
+    $,
+    choice(
+      alias($._operator_qual_dot_head, $.operator),
+      $._ops,
+    ),
+    field('right_operand', $.expression),
+  ),
+
+  _exp_negation: $ => seq(
+    field('minus', '-'),
+    prec('negation', field('expression', $.expression)),
+  ),
+
+  /**
+   * Infix expressions have severe conflicts with several structures:
+   *
+   * - Negation is supposed to bind less tight than application and tighter than infix, which requires an unsolvable
+   *   precedence configuration
+   * - Qualified operators cannot be identified with single-token lookahead, which causes ambiguity with function
+   *   application
+   * - Left operator sections require infix expressions in their operand to reduce before the section operator, but
+   *   single-token lookahead also makes this decision impossible
+   *
+   * All of these are solved with external symbols.
+   * Consult `grammar/externals.js` for more information.
+   */
+  _exp_infix: $ => prec.right('infix', seq(
+    field('left_operand', $.expression),
+    optional($._cond_no_section_op),
+    field('operator', choice(
+      seq($._cond_minus, $._operator_minus),
+      $._exp_op,
+      seq($._cond_qualified_op, $._qsym),
+    )),
+    field('right_operand', $.expression),
+  )),
+
+  // ------------------------------------------------------------------------
+  // top level
+  // ------------------------------------------------------------------------
+
+  /**
+   * This is a supertype.
+   */
+  expression: $ => choice(
+    alias($._exp_infix, $.infix),
+    alias($._exp_negation, $.negation),
+    alias($._exp_apply, $.apply),
+    alias($._exp_record, $.record),
+    alias($._exp_projection, $.projection),
+    alias($._exp_arithmetic_sequence, $.arithmetic_sequence),
+    alias($._exp_list_comprehension, $.list_comprehension),
+    alias($._exp_unboxed_tuple, $.unboxed_tuple),
+    alias($._exp_unboxed_sum, $.unboxed_sum),
+    alias($._exp_projection_selector, $.projection_selector),
+    alias($._exp_quote, $.quote),
+    alias($._exp_typed_quote, $.typed_quote),
+    alias($._exp_th_quoted_name, $.th_quoted_name),
+    alias($._exp_lambda_case, $.lambda_case),
+    alias($._exp_lambda_cases, $.lambda_cases),
+    alias($._exp_do, $.do),
+    alias($._exp_parens, $.parens),
+    alias($._exp_tuple, $.tuple),
+    alias($._exp_list, $.list),
+    // Not using a name like "empty list" here because it's only really special in types.
+    alias($._plist, $.list),
+    alias($._exp_section_left, $.left_section),
+    alias($._exp_section_right, $.right_section),
+    $._exp_greedy,
+    alias($._exp_case, $.case),
+    alias($._exp_multi_way_if, $.multi_way_if),
+    $._exp_name,
+    $._universal,
+  ),
+
+  _exp_signature: $ => prec.right('annotated', seq(
+    field('expression', $.expression),
+    $._type_annotation,
+  )),
+
+  _exp: $ => choice(
+    alias($._exp_signature, $.signature),
+    // Right-associative means that the reduction of `expression` to `_exp` loses against any shift.
+    prec.right($.expression),
+  ),
+
+}

package/grammar/externals.js

@@ -0,0 +1,177 @@
+module.exports = {
+
+  /**
+   * These rules are handled manually by the custom lexer in `src/scanner.c`.
+   * Whenever the symbols are used in the rule tree, the parser executes the scanner.
+   * Since several symbols are present both here and in `extras`, the scanner will be invoked before every single
+   * natively lexed symbol, repeatedly, until it fails.
+   * This makes indentation/layout tracking simpler.
+   *
+   * There are three special behavioral classes of symbols in this grammar, indicated by their name prefix:
+   *
+   * `cmd`: Commands are sequenced after key tokens like opening braces that unconditionally trigger a state change in
+   *        the scanner.
+   *        They don't require the scanner to _decide_ that a symbol is valid – the native parser has already committed
+   *        to a symbol, and the command relays that information to the scanner so it can update its state and return.
+   *        For example, after having parsed an opening brace, the scanner adds a brace context to the stack in reaction
+   *        to `_cond_brace_open`, which will influence its future behavior.
+   *        These symbols do not produce nodes in the parse tree.
+   *
+   * `cond`: Conditions are hard requirements in the grammar that are decided by the scanner based on its state and the
+   *         lookahead.
+   *         For example, whitespace-sensitive operators (tight infix /prefix ops) like the `@` in a type application or
+   *         the `.` in a record projection are lexed when the following characters have certain properties.
+   *         When the scanner decides that such a symbol is valid, the parser cannot attempt to use an alternative
+   *         interpretation like it can with conflicts.
+   *         This is often difficult, but also especially useful to force declarations and other layout items to be
+   *         terminated when a new line has smaller or equal indent, preventing incorrect interpretations as top level
+   *         expression splices spanning multiple lines.
+   *
+   * `phantom`: Phantom symbols are used to signal to the scanner that certain constructs are valid without requiring
+   *            the scanner to actually parse them and produce text nodes for them.
+   *            In the grammar, they are always optional, and the scanner never emits the phantom symbol itself, but
+   *            decides whether to emit _other_ symbols when the phantom symbol is _not_ valid.
+   *            This is used to implement GHC's tactic of allowing layouts to end _whenever a parse error occurs_, but
+   *            using specialized heuristics.
+   *
+   * Most other symbols produce nodes, except for newline and the error sentinel.
+   */
+  externals: $ => [
+
+    // This is an unused symbol that indicates to the scanner that a parse error has occurred.
+    // Tree-sitter marks _all_ symbols as valid when calling the scanner after an error, so a symbol that's not used in
+    // the grammar is only valid if that is the case.
+    $.error_sentinel,
+
+    // Emitted after every newline with equal or less indent to terminate layout-based rules, with a few exceptions.
+    $._cond_layout_semicolon,
+
+    // Instruct the scanner to push a layout context.
+    // The first one is for declarations and miscellaneous other layouts.
+    // The others are for the specific syntax construct.
+    // The quote layout is for declaration quotes (`[d|data A ...|]`).
+    $._cmd_layout_start,
+    $._cmd_layout_start_do,
+    $._cmd_layout_start_case,
+    $._cmd_layout_start_if,
+    $._cmd_layout_start_let,
+    $._cmd_layout_start_quote,
+    // This variant is used in a `choice` with the others, and serves only to create a terminal node for explicit
+    // braces.
+    // If the scanner emitted the same symbol for virtual and explicit braces, we would either get an anonymous node
+    // ranging over the brace, or a terminal brace node even for virtual starts if we were to alias the symbol to '{'
+    // unconditionally.
+    // So we use separate symbols and alias only this one.
+    // The same reasoning applies to `_cond_layout_end_explicit`.
+    // The terminal could be ensured in different ways – adding an `optional('{')` after the start symbol, using
+    // `seq($._cmd_layout_start_explicit, '{')` instead of including the brace in the scanner range, or branching the
+    // entire layout on the start token to unconditionally use `_cmd_brace_close` instead of
+    // `_cond_layout_end_explicit`.
+    // However, these solutions are all very expensive, adding between 500 and 1000kB to the shared object size, and up
+    // to a second in generation time.
+    $._cmd_layout_start_explicit,
+
+    // Emitted when a new line's indent mandates ending the current layout (depending on the layout sort), or when a
+    // special inline layout-ending token is encountered, like an `in`.
+    $._cond_layout_end,
+    $._cond_layout_end_explicit,
+
+    // Instruct the scanner to push or pop a brace context.
+    $._cmd_brace_open,
+    $._cmd_brace_close,
+
+    // Instruct the scanner to push or pop a tuple expression context.
+    // In parenthesized or bracketed expressions, certain tokens (like commas, bars, and closing brackets), can end
+    // layouts, so the scanner must be aware of them.
+    $._cmd_texp_start,
+    $._cmd_texp_end,
+
+    // Signal to the scanner that these symbols are valid.
+    // See the explanation of phantom symbols above.
+    $._phantom_where,
+    $._phantom_in,
+    $._phantom_arrow,
+    $._phantom_bar,
+    $._phantom_deriving,
+
+    // Detect and emit text nodes for comments and CPP.
+    // In particular, #else branches of CPP conditionals a fully contained in the resulting node, since their nonlinear
+    // nature means they cannot be parsed.
+    $.comment,
+    $.haddock,
+    $.cpp,
+    $.pragma,
+
+    // Starting quote brackets are a bit messy, so it's easier to let the scanner signal that it encountered one.
+    // The body produces a text node until the ending bracket.
+    $._cond_quote_start,
+    $.quasiquote_body,
+
+    // Whitespace-sensitive operators for splices (`$`, `$$`), projection and module dots (`a.b`, `A.b`), arithmetic
+    // sequence dots `[1..10]`, as-pattern (`a@(Just b)`), type application (`@Int`), strictness and laziness
+    // annotations (`!pat`, `~Int`), and modifiers (`Int %1 -> Int`).
+    $._cond_splice,
+    $._cond_qual_dot,
+    $._cond_tight_dot,
+    $._cond_prefix_dot,
+    $._cond_dotdot,
+    $._cond_tight_at,
+    $._cond_prefix_at,
+    $._cond_tight_bang,
+    $._cond_prefix_bang,
+    $._cond_tight_tilde,
+    $._cond_prefix_tilde,
+    $._cond_prefix_percent,
+
+    // GHC lexes all qualified names as atomic tokens, but we can't do that because we need nodes for their
+    // substructures.
+    // However, infix expressions need single-token lookahead for the operator to resolve conflicts, which is
+    // impossible without an external.
+    // This symbol detects a qualified symbolic operator.
+    $._cond_qualified_op,
+    // This one performs an additional lookahead check for a following closing parenthesis, to disambiguate left
+    // sections from infix ops in `(a - b +)`.
+    $._cond_left_section_op,
+    // This is an auxiliary for `_cond_left_section_op` that allows restarting the scanner without that symbol being
+    // valid again when whitespace was skipped after a symop to discover that there's no parenthesis.
+    // Without this, we would get wrong token ranges for the operator.
+    $._cond_no_section_op,
+
+    // This symbol always succeeds when a minus is ahead.
+    // Infix expressions with minus as the operator conflict with function application.
+    // `a - b` can be parsed as `a (- b)`, which would normally be solved by a simple precedence entry, but is
+    // impossible to express because it contradicts another conflict: Negation in the left operand, `- a + b`, which
+    // must be parsed as `(- a) + b`.
+    // Simply sequencing this external before the operator solves this conflict, because a minus following an expression
+    // can never be negation; it can only occur at the beginning of an expression.
+    $._cond_minus,
+
+    /**
+     * The following symbols perform lookahead for various type constructs to resolve conflicts between infix types,
+     * contexts, and type/datacon heads, because using GLR conflicts for these is very brittle and frequently leads to
+     * misparsed trees.
+     *
+     * See the documentation under 'Constraints' in `src/scanner.c` for more.
+     */
+    $._cond_context,
+    $._cond_infix,
+    $._cond_data_infix,
+    $._cond_assoc_tyinst,
+
+    // Symbolic operators, producing text nodes.
+    // These are very difficult to parse in the grammar, because unlike most languages, Haskell's operators are not a
+    // finite set, and therefore cannot be lexically disambiguated from special operators like `->`.
+    // In the presence of runtime conflicts, this can easily lead to the invalid interpretation of reserved operators as
+    // identifiers.
+    // Furthermore, the regexes for all the unicode categories produce very large ternary operator trees if specified
+    // in the grammar.
+    $._varsym,
+    $._consym,
+
+    // The newline is used as a dummy symbol that is emitted whenever the scanner has to update its state even though
+    // the current position must be parsed by the grammar.
+    token(prec(0, /\n/)),
+
+  ],
+
+}

package/grammar/general.js

@@ -0,0 +1,54 @@
+const {
+  sep1,
+  layout_sort,
+} = require('./util.js')
+
+module.exports = {
+
+  // ------------------------------------------------------------------------
+  // guard
+  // ------------------------------------------------------------------------
+
+  generator: $ => seq(
+    field('pattern', $._pat),
+    field('arrow', $._larrow),
+    field('expression', $._exp),
+  ),
+
+  _let_binds: $ => layout_sort($, $._cmd_layout_start_let, field('decl', $.decl)),
+
+  _let: $ => seq('let', optional(field('binds', alias($._let_binds, $.local_binds)))),
+
+  let: $ => $._let,
+
+  /**
+   * This is a supertype.
+   */
+  guard: $ => choice(
+    // Cannot be named `pattern` because name clash.
+    alias($.generator, $.pattern_guard),
+    $.let,
+    alias($._exp, $.boolean),
+  ),
+
+  guards: $ => sep1(',', field('guard', $.guard)),
+
+  _guards: $ => seq(
+    $._bar,
+    $._cmd_texp_start,
+    field('guards', $.guards),
+  ),
+
+  // ------------------------------------------------------------------------
+  // rules shared by expression, pattern, type
+  // ------------------------------------------------------------------------
+
+  _universal: $ => choice(
+    $.splice,
+    $.quasiquote,
+    $.literal,
+    $._unit_cons,
+    $._tuple_cons,
+  ),
+
+}