@algosail/lang 0.2.11 → 0.5.0

package/lib/parse/parse-source.js

@@ -0,0 +1,912 @@
+/**
+ * Phrase parser RFC-0.1 §12.2 (L1) + построение AST (ast-spec.js).
+ */
+
+import { createLexerState, nextToken, readPathToken, skipWs } from './lexer.js'
+import * as diag from './diagnostics.js'
+import * as ab from './ast-build.js'
+
+function skipInlineWs (state) {
+  while (state.i < state.source.length) {
+    const c = state.source[state.i]
+    if (c === ' ' || c === '\t') {
+      state.i++
+      state.col++
+    } else break
+  }
+}
+
+function createCtx (state) {
+  return {
+    state,
+    diagnostics: [],
+    regexpOk: false,
+    buf: null,
+    pushback: [],
+    items: [],
+    pendingDoc: []
+  }
+}
+
+function peek (ctx) {
+  if (ctx.pushback.length) return ctx.pushback[ctx.pushback.length - 1]
+  if (ctx.buf === null) ctx.buf = nextToken(ctx.state, ctx)
+  return ctx.buf
+}
+
+function eat (ctx) {
+  if (ctx.pushback.length) return ctx.pushback.pop()
+  const t = peek(ctx)
+  ctx.buf = null
+  return t
+}
+
+function unget (ctx, t) {
+  ctx.pushback.push(t)
+}
+
+function pushE1001 (ctx, t, tokenText) {
+  ctx.diagnostics.push({
+    code: 'E1001',
+    message: diag.e1001UnexpectedToken(tokenText),
+    offset: t.offset,
+    line: t.line,
+    column: t.column
+  })
+}
+
+function pushDiag (ctx, code, message, t) {
+  ctx.diagnostics.push({
+    code,
+    message,
+    offset: t.offset,
+    line: t.line,
+    column: t.column
+  })
+}
+
+function tokenSpan (ctx, tok) {
+  const sp = ab.openSpan(tok)
+  ab.closeSpan(ctx.state, sp)
+  return sp
+}
+
+function drainComments (ctx) {
+  const out = []
+  while (peek(ctx).type === 'COMMENT') out.push(eat(ctx).value)
+  return out
+}
+
+function forkCtx (ctx) {
+  const state = {
+    source: ctx.state.source,
+    i: ctx.state.i,
+    line: ctx.state.line,
+    col: ctx.state.col
+  }
+  const sub = createCtx(state)
+  sub.regexpOk = ctx.regexpOk
+  return sub
+}
+
+/**
+ * `(` … парная `)` содержит хотя бы один `->` при depth === 1 внутри этой пары
+ * (стрелка «между In и Out» этой скобочной группы, не внутри вложенных `(` … `)`).
+ * Тогда это quotation_sig / quotation_type; иначе — paren_type + type_expr.
+ */
+function sigParenHasTopLevelArrow (ctx) {
+  const t = peek(ctx)
+  if (t.type !== '(') return false
+  const sub = forkCtx(ctx)
+  // peek() мог оставить buf и продвинуть state.i за «(»; fork иначе начнёт не с той скобки.
+  sub.state.i = t.offset
+  sub.state.line = t.line
+  sub.state.col = t.column
+  sub.buf = null
+  eat(sub)
+  let depth = 1
+  let sawArrowAtDepth1 = false
+  while (depth > 0 && peek(sub).type !== 'EOF') {
+    const u = eat(sub)
+    if (u.type === '(') depth++
+    else if (u.type === ')') depth--
+    else if (u.type === 'ARROW' && depth === 1) sawArrowAtDepth1 = true
+  }
+  return sawArrowAtDepth1
+}
+
+function skipBodyComments (ctx) {
+  while (peek(ctx).type === 'COMMENT') eat(ctx)
+}
+
+function wrapSigType (inner) {
+  return ab.node('sig_type_expr', inner.span, { type: inner })
+}
+
+function effectNameFromAdd (t) {
+  const s = String(t.value)
+  return s.startsWith('+') ? s.slice(1) : s
+}
+
+function effectNameFromSub (t) {
+  const s = String(t.value)
+  return s.startsWith('-') ? s.slice(1) : s
+}
+
+function sigTypeArgFollows (ctx) {
+  const t = peek(ctx)
+  return (
+    t.type === '(' ||
+    t.type === 'MODULE_TYPE_REF' ||
+    t.type === 'TYPE_NAME' ||
+    t.type === 'LOWER_IDENT'
+  )
+}
+
+function typeExprDelimiter (ctx) {
+  const t = peek(ctx)
+  return (
+    t.type === 'EOF' ||
+    t.type === ')' ||
+    t.type === '|' ||
+    t.type === 'ARROW' ||
+    t.type === 'MODULE' ||
+    t.type === 'AMP' ||
+    t.type === 'WORD_DEF' ||
+    t.type === 'COMMENT'
+  )
+}
+
+/**
+ * TypeName без параметров на слоте операнда `List` / `Dict` / `Map` (RFC-0.1 list_type, dict_type, map_type):
+ * иначе `Map Str Num` ошибочно разбиралось как `Map (Str Num)`.
+ * Параметризованные формы (`Maybe a`, `Vec T`) и группировка — через полный `buildTypeExpr` или `( … )`.
+ */
+const PRIMITIVE_TYPE_NAMES = new Set([
+  'Str', 'String', 'Num', 'Int', 'Bool', 'Float', 'Double', 'Char', 'Unit', 'Nil'
+])
+
+/** @returns {object | null} AstTypeExpr */
+function buildTypeExprCombinatorOperand (ctx) {
+  const t = peek(ctx)
+  if (t.type === '(') {
+    const o = eat(ctx)
+    const sp = ab.openSpan(o)
+    const inner = buildTypeExpr(ctx)
+    if (!inner) return null
+    if (peek(ctx).type !== ')') {
+      const u = peek(ctx)
+      pushE1001(ctx, u, String(u.value || u.type))
+      return null
+    }
+    eat(ctx)
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('paren_type', sp, { inner })
+  }
+  if (t.type === 'MODULE_TYPE_REF') {
+    const m = eat(ctx)
+    return ab.node('module_type_ref', tokenSpan(ctx, m), { module: m.module, type: m.typeName })
+  }
+  if (t.type === 'LOWER_IDENT') {
+    const id = eat(ctx)
+    const sp = tokenSpan(ctx, id)
+    return ab.node('type_var', sp, { name: String(id.value) })
+  }
+  if (t.type !== 'TYPE_NAME') {
+    pushE1001(ctx, t, String(t.value || t.type))
+    return null
+  }
+  const nm = String(t.value)
+  if (PRIMITIVE_TYPE_NAMES.has(nm)) {
+    const idtok = eat(ctx)
+    return ab.node('type_name', tokenSpan(ctx, idtok), { name: nm })
+  }
+  return buildTypeExpr(ctx)
+}
+
+/** @returns {object | null} AstTypeExpr */
+function buildTypeExpr (ctx) {
+  const t = peek(ctx)
+  if (t.type === '(') {
+    const o = eat(ctx)
+    const sp = ab.openSpan(o)
+    const inner = buildTypeExpr(ctx)
+    if (!inner) return null
+    if (peek(ctx).type !== ')') {
+      const u = peek(ctx)
+      pushE1001(ctx, u, String(u.value || u.type))
+      return null
+    }
+    eat(ctx)
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('paren_type', sp, { inner })
+  }
+  if (t.type === 'MODULE_TYPE_REF') {
+    const m = eat(ctx)
+    return ab.node('module_type_ref', tokenSpan(ctx, m), { module: m.module, type: m.typeName })
+  }
+  if (t.type !== 'LOWER_IDENT' && t.type !== 'TYPE_NAME') {
+    pushE1001(ctx, t, String(t.value || t.type))
+    return null
+  }
+  const first = eat(ctx)
+  const sp = ab.openSpan(first)
+  if (first.value === 'List') {
+    const arg = buildTypeExprCombinatorOperand(ctx)
+    if (!arg) return null
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('type_app', sp, { ctor: 'List', args: [arg] })
+  }
+  if (first.value === 'Dict') {
+    const arg = buildTypeExprCombinatorOperand(ctx)
+    if (!arg) return null
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('type_app', sp, { ctor: 'Dict', args: [arg] })
+  }
+  if (first.value === 'Map') {
+    const a = buildTypeExprCombinatorOperand(ctx)
+    if (!a) return null
+    const b = buildTypeExprCombinatorOperand(ctx)
+    if (!b) return null
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('type_app', sp, { ctor: 'Map', args: [a, b] })
+  }
+  if (first.type === 'TYPE_NAME') {
+    const args = []
+    while (!typeExprDelimiter(ctx) && sigTypeArgFollows(ctx)) {
+      const a = buildTypeExpr(ctx)
+      if (!a) return null
+      args.push(a)
+    }
+    ab.closeSpan(ctx.state, sp)
+    if (args.length === 0) return ab.node('type_name', sp, { name: first.value })
+    return ab.node('type_app', sp, { ctor: first.value, args })
+  }
+  ab.closeSpan(ctx.state, sp)
+  return ab.node('type_var', sp, { name: first.value })
+}
+
+/** @returns {object | null} AstTypeExpr (core of sig_type_expr) */
+function buildSigTypeExprCore (ctx) {
+  const t = peek(ctx)
+  if (t.type === '(') {
+    if (sigParenHasTopLevelArrow(ctx)) {
+      const q = parseQuotSigAst(ctx)
+      if (!q) return null
+      return ab.node('quotation_type', q.node.span, { inner: q.node.inner })
+    }
+    const o = eat(ctx)
+    const sp = ab.openSpan(o)
+    const inner = buildTypeExpr(ctx)
+    if (!inner) return null
+    if (peek(ctx).type !== ')') {
+      const u = peek(ctx)
+      pushE1001(ctx, u, String(u.value || u.type))
+      return null
+    }
+    eat(ctx)
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('paren_type', sp, { inner })
+  }
+  if (t.type === 'MODULE_TYPE_REF') {
+    const m = eat(ctx)
+    return ab.node('module_type_ref', tokenSpan(ctx, m), { module: m.module, type: m.typeName })
+  }
+  if (t.type !== 'LOWER_IDENT' && t.type !== 'TYPE_NAME') {
+    pushE1001(ctx, t, String(t.value || t.type))
+    return null
+  }
+  const first = eat(ctx)
+  const sp = ab.openSpan(first)
+  if (first.value === 'List') {
+    const arg = buildTypeExprCombinatorOperand(ctx)
+    if (!arg) return null
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('type_app', sp, { ctor: 'List', args: [arg] })
+  }
+  if (first.value === 'Dict') {
+    const arg = buildTypeExprCombinatorOperand(ctx)
+    if (!arg) return null
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('type_app', sp, { ctor: 'Dict', args: [arg] })
+  }
+  if (first.value === 'Map') {
+    const a = buildTypeExprCombinatorOperand(ctx)
+    if (!a) return null
+    const b = buildTypeExprCombinatorOperand(ctx)
+    if (!b) return null
+    ab.closeSpan(ctx.state, sp)
+    return ab.node('type_app', sp, { ctor: 'Map', args: [a, b] })
+  }
+  if (first.type === 'TYPE_NAME') {
+    const head = String(first.value)
+    // Примитив — один слот; соседний `TYPE_NAME` — следующий sig_item, не аргумент (RFC-0.1: скобки только у параметризованных форм).
+    if (PRIMITIVE_TYPE_NAMES.has(head)) {
+      ab.closeSpan(ctx.state, sp)
+      return ab.node('type_name', sp, { name: head })
+    }
+    const args = []
+    while (sigTypeArgFollows(ctx)) {
+      const a = buildSigTypeExprCore(ctx)
+      if (!a) return null
+      args.push(a)
+    }
+    ab.closeSpan(ctx.state, sp)
+    if (args.length === 0) return ab.node('type_name', sp, { name: first.value })
+    return ab.node('type_app', sp, { ctor: first.value, args })
+  }
+  ab.closeSpan(ctx.state, sp)
+  return ab.node('type_var', sp, { name: first.value })
+}
+
+function parseSigItemInto (ctx, side, stack, effectsAdd, effectsRemove) {
+  const t = peek(ctx)
+  if (t.type === 'COMMENT') {
+    pushE1001(ctx, t, 'comment')
+    return false
+  }
+  if (t.type === 'EFFECT_ADD') {
+    const e = eat(ctx)
+    if (side === 'left') {
+      pushDiag(ctx, 'E1007', diag.e1007EffectOnlyAfterArrow(String(e.value)), e)
+      return false
+    }
+    const sp = tokenSpan(ctx, e)
+    effectsAdd.push(ab.node('sig_effect_add', sp, { name: effectNameFromAdd(e), side }))
+    return true
+  }
+  if (t.type === 'EFFECT_SUB') {
+    const e = eat(ctx)
+    if (side === 'left') {
+      pushDiag(ctx, 'E1007', diag.e1007EffectOnlyAfterArrow(String(e.value)), e)
+      return false
+    }
+    const sp = tokenSpan(ctx, e)
+    effectsRemove.push(ab.node('sig_effect_remove', sp, { name: effectNameFromSub(e), side }))
+    return true
+  }
+  if (t.type === 'STACK_VAR') {
+    const s = eat(ctx)
+    const sp = tokenSpan(ctx, s)
+    stack.push(ab.node('stack_var', sp, { name: String(s.value) }))
+    return true
+  }
+  if (t.type === '(') {
+    if (sigParenHasTopLevelArrow(ctx)) {
+      const q = parseQuotSigAst(ctx)
+      if (!q) return false
+      stack.push(q.node)
+      return true
+    }
+    const o = eat(ctx)
+    const sp = ab.openSpan(o)
+    const inner = buildTypeExpr(ctx)
+    if (!inner) return false
+    if (peek(ctx).type !== ')') {
+      const u = peek(ctx)
+      pushE1001(ctx, u, String(u.value || u.type))
+      return false
+    }
+    eat(ctx)
+    ab.closeSpan(ctx.state, sp)
+    stack.push(wrapSigType(ab.node('paren_type', sp, { inner })))
+    return true
+  }
+  if (t.type === 'MODULE_TYPE_REF') {
+    const inner = buildSigTypeExprCore(ctx)
+    if (!inner) return false
+    stack.push(wrapSigType(inner))
+    return true
+  }
+  if (t.type !== 'LOWER_IDENT' && t.type !== 'TYPE_NAME') {
+    pushE1001(ctx, t, String(t.value || t.type))
+    return false
+  }
+  const a = eat(ctx)
+  if (peek(ctx).type === ':') {
+    const prefixSp = tokenSpan(ctx, a)
+    eat(ctx)
+    const res = parseQuotSigAst(ctx)
+    if (!res) return false
+    const sp = ab.mergeSpan(prefixSp, res.node.span)
+    const named = ab.node('named_quotation_sig', sp, { prefix: String(a.value), quotation: res.node })
+    stack.push(named)
+    return true
+  }
+  if (
+    a.type === 'TYPE_NAME' &&
+    String(a.value) === 'Quote' &&
+    peek(ctx).type === '('
+  ) {
+    const quoteSp = tokenSpan(ctx, a)
+    const q = parseQuotSigAst(ctx)
+    if (!q) return false
+    const sp = ab.mergeSpan(quoteSp, q.node.span)
+    stack.push(ab.node('quotation_sig', sp, { inner: q.node.inner }))
+    return true
+  }
+  unget(ctx, a)
+  const inner = buildSigTypeExprCore(ctx)
+  if (!inner) return false
+  stack.push(wrapSigType(inner))
+  return true
+}
+
+/** @returns {{ node: object, sawOuterArrow: boolean } | null} */
+function parseQuotSigAst (ctx) {
+  const o = eat(ctx)
+  if (o.type !== '(') {
+    unget(ctx, o)
+    return null
+  }
+  const sp = ab.openSpan(o)
+  const left = []
+  const right = []
+  const effectsAdd = []
+  const effectsRemove = []
+  const half = (side, st) => {
+    while (peek(ctx).type !== 'ARROW' && peek(ctx).type !== ')') {
+      if (!parseSigItemInto(ctx, side, st, effectsAdd, effectsRemove)) return false
+    }
+    return true
+  }
+  if (!half('left', left)) return null
+  let sawOuterArrow = false
+  if (peek(ctx).type === 'ARROW') {
+    sawOuterArrow = true
+    eat(ctx)
+    if (!half('right', right)) return null
+  }
+  if (peek(ctx).type !== ')') {
+    const u = peek(ctx)
+    pushE1001(ctx, u, String(u.value || u.type))
+    return null
+  }
+  const c = eat(ctx)
+  ab.closeSpan(ctx.state, sp)
+  const inner = ab.node('signature', sp, {
+    left,
+    right,
+    effectsAdd,
+    effectsRemove
+  })
+  const node = ab.node('quotation_sig', sp, { inner })
+  return { node, sawOuterArrow }
+}
+
+function parseParenSignatureAst (ctx) {
+  const res = parseQuotSigAst(ctx)
+  if (!res) return null
+  const { node, sawOuterArrow } = res
+  const sig = node.inner
+  // RFC-0.1 §6.1.1: лишняя пара скобок вокруг sugar-цитаты `( ( In -> Out ) )` не нормируется;
+  // без верхнего `->` у слова это один stack-item `quotation_sig` — отклоняем (E1001).
+  if (
+    !sawOuterArrow &&
+    sig.left.length === 1 &&
+    sig.right.length === 0 &&
+    sig.left[0].kind === 'quotation_sig'
+  ) {
+    const st = sig.left[0].span.start
+    pushE1001(ctx, st, '(')
+    return null
+  }
+  return sig
+}
+
+function parseSignatureAst (ctx) {
+  return parseParenSignatureAst(ctx)
+}
+
+function parseImportAst (ctx) {
+  const modTok = eat(ctx)
+  ctx.regexpOk = false
+  ctx.buf = null
+  const sp = ab.openSpan(modTok)
+  const moduleName = String(modTok.value)
+  let bracket = null
+  const st = ctx.state
+  skipWs(st)
+  if (st.i < st.source.length && st.source[st.i] === '(') {
+    bracket = { words: [], types: [] }
+    st.i += 1
+    st.col += 1
+    while (true) {
+      skipWs(st)
+      if (st.i >= st.source.length) {
+        pushE1001(ctx, { type: 'EOF', offset: st.i, line: st.line, column: st.col }, 'EOF')
+        return null
+      }
+      if (st.source[st.i] === ')') {
+        st.i += 1
+        st.col += 1
+        break
+      }
+      ctx.buf = null
+      const u = peek(ctx)
+      if (u.type === 'WORD_DEF') {
+        const wd = eat(ctx)
+        bracket.words.push(String(wd.value))
+      } else if (u.type === 'AMP') {
+        eat(ctx)
+        const tn = eat(ctx)
+        if (tn.type !== 'TYPE_NAME') {
+          pushE1001(ctx, tn, String(tn.value || tn.type))
+          return null
+        }
+        bracket.types.push(String(tn.value))
+      } else {
+        pushE1001(ctx, u, String(u.value || u.type))
+        return null
+      }
+    }
+  }
+  const pathTok = readPathToken(ctx.state, ctx)
+  ctx.buf = null
+  if (!pathTok) {
+    ctx.diagnostics.push({
+      code: 'E1003',
+      message: diag.e1003BadImportPath(''),
+      offset: modTok.offset,
+      line: modTok.line,
+      column: modTok.column
+    })
+    return null
+  }
+  const src = ctx.state.source
+  skipInlineWs(ctx.state)
+  if (ctx.state.i < src.length) {
+    const ch = src[ctx.state.i]
+    if (ch !== '\n' && ch !== '\r') {
+      const lineEnd = src.indexOf('\n', ctx.state.i)
+      const end = lineEnd === -1 ? src.length : lineEnd
+      const bad = src.slice(pathTok.offset, end).replace(/\s+$/, '')
+      ctx.diagnostics.push({
+        code: 'E1003',
+        message: diag.e1003BadImportPath(bad),
+        offset: pathTok.offset,
+        line: pathTok.line,
+        column: pathTok.column
+      })
+      ctx.state.i = end
+      return null
+    }
+  }
+  ab.closeSpan(ctx.state, sp)
+  const doc = ctx.pendingDoc ? [...ctx.pendingDoc] : []
+  return ab.node('import', sp, {
+    module: moduleName,
+    path: String(pathTok.value),
+    bracket,
+    doc
+  })
+}
+
+function parseTypeDeclAst (ctx) {
+  if (peek(ctx).type !== 'AMP') return null
+  const amp = eat(ctx)
+  const declSp = ab.openSpan(amp)
+  const name = eat(ctx)
+  if (name.type !== 'TYPE_NAME') {
+    pushE1001(ctx, name, String(name.value || name.type))
+    return null
+  }
+  const typeParams = []
+  while (peek(ctx).type === 'LOWER_IDENT' || peek(ctx).type === 'TYPE_NAME') {
+    const p = eat(ctx)
+    typeParams.push(String(p.value))
+  }
+  const headerDoc = drainComments(ctx)
+  const u = peek(ctx)
+  if (u.type === '|') {
+    const tags = []
+    while (peek(ctx).type === '|') {
+      eat(ctx)
+      const tag = eat(ctx)
+      if (tag.type !== 'TYPE_NAME') {
+        pushE1001(ctx, tag, String(tag.value || tag.type))
+        return null
+      }
+      const tagSp = ab.openSpan(tag)
+      let payload = null
+      const v = peek(ctx)
+      if (
+        v.type !== '|' &&
+        v.type !== 'EOF' &&
+        v.type !== 'AMP' &&
+        v.type !== 'WORD_DEF' &&
+        v.type !== 'MODULE' &&
+        v.type !== 'COMMENT'
+      ) {
+        payload = buildTypeExpr(ctx)
+        if (!payload) return null
+        tagSp.end = payload.span.end
+      }
+      const tagDoc = drainComments(ctx)
+      ab.closeSpan(ctx.state, tagSp)
+      tags.push({
+        name: String(tag.value),
+        payload,
+        span: tagSp,
+        doc: tagDoc
+      })
+    }
+    ab.closeSpan(ctx.state, declSp)
+    return ab.node('sum_type', declSp, {
+      name: String(name.value),
+      typeParams,
+      doc: headerDoc,
+      tags
+    })
+  }
+  if (u.type === ':') {
+    const fields = []
+    while (peek(ctx).type === ':') {
+      eat(ctx)
+      const fn = eat(ctx)
+      if (fn.type !== 'LOWER_IDENT') {
+        pushE1001(ctx, fn, String(fn.value || fn.type))
+        return null
+      }
+      const fnSp = tokenSpan(ctx, fn)
+      const ty = buildTypeExpr(ctx)
+      if (!ty) return null
+      const fieldDoc = drainComments(ctx)
+      const fsp = ab.mergeSpan(fnSp, ty.span)
+      fields.push({
+        name: String(fn.value),
+        type: ty,
+        span: fsp,
+        doc: fieldDoc
+      })
+    }
+    ab.closeSpan(ctx.state, declSp)
+    return ab.node('product_type', declSp, {
+      name: String(name.value),
+      typeParams,
+      doc: headerDoc,
+      fields
+    })
+  }
+  pushE1001(ctx, u, String(u.value || u.type))
+  return null
+}
+
+function litKindFromToken (t) {
+  if (t.type === 'STRING') return 'string'
+  if (t.type === 'NUMBER') return 'number'
+  if (t.type === 'BIGINT') return 'bigint'
+  if (t.type === 'REGEXP') return 'regexp'
+  if (t.type === 'NIL') return 'nil'
+  if (t.type === 'TRUE' || t.type === 'FALSE') return 'bool'
+  return 'string'
+}
+
+function isLiteralToken (t) {
+  return (
+    t.type === 'STRING' ||
+    t.type === 'NUMBER' ||
+    t.type === 'BIGINT' ||
+    t.type === 'REGEXP' ||
+    t.type === 'NIL' ||
+    t.type === 'TRUE' ||
+    t.type === 'FALSE'
+  )
+}
+
+function parseListLiteralAst (ctx) {
+  const o = eat(ctx)
+  const sp = ab.openSpan(o)
+  const elements = []
+  while (peek(ctx).type !== ']') {
+    if (peek(ctx).type === 'EOF') return null
+    if (peek(ctx).type === 'COMMENT') {
+      pushE1001(ctx, peek(ctx), 'comment')
+      return null
+    }
+    ctx.regexpOk = true
+    const t = peek(ctx)
+    if (!isLiteralToken(t)) {
+      pushE1001(ctx, t, String(t.value || t.type))
+      return null
+    }
+    const lt = eat(ctx)
+    elements.push(ab.node('literal', tokenSpan(ctx, lt), { litKind: litKindFromToken(lt), raw: String(lt.value) }))
+  }
+  eat(ctx)
+  ab.closeSpan(ctx.state, sp)
+  return ab.node('list_literal', sp, { elements })
+}
+
+function parseExecutableQuotAst (ctx) {
+  const o = eat(ctx)
+  const sp = ab.openSpan(o)
+  const body = []
+  while (peek(ctx).type !== ')') {
+    if (peek(ctx).type === 'EOF') return null
+    if (peek(ctx).type === 'ARROW') {
+      const a = eat(ctx)
+      pushDiag(ctx, 'E1005', diag.e1005ArrowInQuotation(), a)
+      return null
+    }
+    skipBodyComments(ctx)
+    if (peek(ctx).type === ')') break
+    ctx.regexpOk = true
+    const step = parseWordStepAst(ctx)
+    if (!step) return null
+    body.push(step)
+  }
+  eat(ctx)
+  ab.closeSpan(ctx.state, sp)
+  return ab.node('quotation', sp, { body })
+}
+
+function parseWordStepAst (ctx) {
+  const t = peek(ctx)
+  if (t.type === 'EFFECT_ADD') {
+    pushDiag(ctx, 'E1006', diag.e1006EffectOutsideSignature(t.value), t)
+    eat(ctx)
+    return null
+  }
+  if (t.type === 'EFFECT_SUB') {
+    pushDiag(ctx, 'E1006', diag.e1006EffectOutsideSignature(t.value), t)
+    eat(ctx)
+    return null
+  }
+  if (t.type === 'WORD_DEF') {
+    pushE1001(ctx, t, '@')
+    return null
+  }
+  if (t.type === '(') return parseExecutableQuotAst(ctx)
+  if (t.type === '[') return parseListLiteralAst(ctx)
+  if (isLiteralToken(t)) {
+    const lt = eat(ctx)
+    return ab.node('literal', tokenSpan(ctx, lt), { litKind: litKindFromToken(lt), raw: String(lt.value) })
+  }
+  if (t.type === 'WORD_REF') {
+    const w = eat(ctx)
+    return ab.node('word_ref', tokenSpan(ctx, w), { name: String(w.value) })
+  }
+  if (t.type === 'MODULE_WORD_REF') {
+    const w = eat(ctx)
+    return ab.node('module_word_ref', tokenSpan(ctx, w), { module: w.module, word: w.word })
+  }
+  if (t.type === 'LOWER_IDENT') {
+    const id = eat(ctx)
+    const sp = tokenSpan(ctx, id)
+    const nm = String(id.value)
+    return ab.node('builtin', sp, { name: nm })
+  }
+  if (t.type === ':') {
+    eat(ctx)
+    const n = eat(ctx)
+    if (n.type !== 'LOWER_IDENT') {
+      pushE1001(ctx, n, String(n.value || n.type))
+      return null
+    }
+    return ab.node('slot_write', tokenSpan(ctx, n), { name: String(n.value) })
+  }
+  if (t.type === ';') {
+    eat(ctx)
+    const n = eat(ctx)
+    if (n.type !== 'LOWER_IDENT') {
+      pushE1001(ctx, n, String(n.value || n.type))
+      return null
+    }
+    return ab.node('slot_read', tokenSpan(ctx, n), { name: String(n.value) })
+  }
+  pushE1001(ctx, t, String(t.value || t.type))
+  return null
+}
+
+function parseWordAst (ctx) {
+  const at = eat(ctx)
+  if (at.type !== 'WORD_DEF') {
+    unget(ctx, at)
+    return null
+  }
+  ctx.regexpOk = false
+  const wspan = ab.openSpan(at)
+  const signature = parseSignatureAst(ctx)
+  if (!signature) {
+    if (ctx.diagnostics.length === 0) {
+      const t = peek(ctx)
+      pushE1001(ctx, t, String(t.value || t.type))
+    }
+    return null
+  }
+  ctx.regexpOk = true
+  const wordDoc = drainComments(ctx)
+  const body = []
+  for (;;) {
+    skipBodyComments(ctx)
+    ctx.regexpOk = true
+    const t = peek(ctx)
+    if (t.type === 'EOF') break
+    if (
+      t.type === 'MODULE' ||
+      t.type === 'AMP' ||
+      t.type === 'WORD_DEF'
+    ) {
+      if (t.column === 1) break
+    }
+    const step = parseWordStepAst(ctx)
+    if (!step) return null
+    body.push(step)
+  }
+  ab.closeSpan(ctx.state, wspan)
+  return ab.node('word', wspan, {
+    name: String(at.value),
+    signature,
+    doc: wordDoc,
+    body
+  })
+}
+
+function parseTopLevelAst (ctx) {
+  const t = peek(ctx)
+  ctx.regexpOk = false
+  if (t.type === 'MODULE') {
+    const n = parseImportAst(ctx)
+    if (n) {
+      ctx.items.push(n)
+      return true
+    }
+    return false
+  }
+  if (t.type === 'AMP') {
+    const n = parseTypeDeclAst(ctx)
+    if (n) {
+      ctx.items.push(n)
+      return true
+    }
+    return false
+  }
+  if (t.type === 'WORD_DEF') {
+    const n = parseWordAst(ctx)
+    if (n) {
+      ctx.items.push(n)
+      return true
+    }
+    return false
+  }
+  if (t.type === 'LOWER_IDENT' || t.type === 'TYPE_NAME') {
+    pushDiag(ctx, 'E1004', diag.e1004UnknownBlock(t.value), t)
+    eat(ctx)
+    return false
+  }
+  pushE1001(ctx, t, String(t.value || t.type))
+  return false
+}
+
+function finishSourceFileAst (ctx) {
+  const end = ab.endPosFromState(ctx.state)
+  const span = {
+    start: { offset: 0, line: 1, column: 1 },
+    end
+  }
+  return ab.node('source_file', span, { items: ctx.items })
+}
+
+export function parseSourceText (sourceText) {
+  const state = createLexerState(sourceText)
+  const ctx = createCtx(state)
+  let pendingDoc = []
+  while (peek(ctx).type !== 'EOF') {
+    pendingDoc = pendingDoc.concat(drainComments(ctx))
+    if (peek(ctx).type === 'EOF') break
+    ctx.pendingDoc = pendingDoc
+    pendingDoc = []
+    const ok = parseTopLevelAst(ctx)
+    ctx.pendingDoc = []
+    if (ctx.diagnostics.length || !ok) break
+  }
+  drainComments(ctx)
+  if (peek(ctx).type !== 'EOF' && ctx.diagnostics.length === 0) {
+    const t = peek(ctx)
+    pushE1001(ctx, t, String(t.value || t.type))
+  }
+  const ast =
+    ctx.diagnostics.length === 0 ? finishSourceFileAst(ctx) : undefined
+  return { diagnostics: ctx.diagnostics, ast }
+}