@algosail/lang 0.2.11 → 0.5.0

package/lib/parse/lexer.js

@@ -0,0 +1,390 @@
+/**
+ * Tokenization aligned with RFC-lex-0.1 (comments, literals, path, regexp vs word_ref).
+ */
+
+import * as diag from './diagnostics.js'
+
+const RE_MODULE = /^\+([A-Z][a-zA-Z0-9_]*)/
+const RE_EFFECT_ADD = /^\+(Async|Fail)\b/
+const RE_EFFECT_SUB = /^-(Async|Fail)\b/
+const RE_WORD_DEF = /^@([a-z][a-zA-Z0-9_]*)/
+const RE_WORD_REF = /^\/([a-z][a-zA-Z0-9_]*)/
+const RE_STACK_VAR = /^~([a-z][a-zA-Z0-9_]*)/
+const RE_MOD_WORD = /^~([A-Z][a-zA-Z0-9_]*)\/([a-z][a-zA-Z0-9_]*)/
+const RE_MOD_TYPE = /^~([A-Z][a-zA-Z0-9_]*)\/([A-Z][a-zA-Z0-9_]*)/
+const RE_IDENT = /^([a-zA-Z_][a-zA-Z0-9_]*)/
+const RE_NUM = /^([0-9][0-9_]*(\.[0-9_]+)?([eE][+-]?[0-9_]+)?|0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+)\b/
+const RE_BIGINT = /^([0-9][0-9_]*|0[xX][0-9a-fA-F_]+)n\b/
+const RE_PATH = /^([^\s()]+)/
+
+function isWs (c) {
+  return c === ' ' || c === '\t' || c === '\n' || c === '\r'
+}
+
+function advancePos (state, n) {
+  for (let k = 0; k < n; k++) {
+    const c = state.source[state.i + k]
+    if (c === '\n') {
+      state.line++
+      state.col = 1
+    } else {
+      state.col++
+    }
+  }
+  state.i += n
+}
+
+export function skipWs (state) {
+  while (state.i < state.source.length && isWs(state.source[state.i])) {
+    advancePos(state, 1)
+  }
+}
+
+function pos (state) {
+  return { offset: state.i, line: state.line, column: state.col }
+}
+
+/**
+ * Блочный комментарий `-- … --` как токен (текст между парами) для doc_block в парсере.
+ * @returns {{ type: 'COMMENT', value: string, offset: number, line: number, column: number } | null}
+ */
+function readCommentToken (state, diagnostics) {
+  const start = pos(state)
+  advancePos(state, 2)
+  const from = state.i
+  const rest = state.source.slice(from)
+  const endRel = rest.indexOf('--')
+  if (endRel === -1) {
+    diagnostics.push({
+      code: 'E1002',
+      message: diag.e1002UnclosedComment(),
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    })
+    state.i = state.source.length
+    return null
+  }
+  const inner = rest.slice(0, endRel)
+  for (let k = 0; k < endRel; k++) {
+    const c = rest[k]
+    if (c === '\n') {
+      state.line++
+      state.col = 1
+    } else {
+      state.col++
+    }
+  }
+  state.i = from + endRel
+  advancePos(state, 2)
+  return {
+    type: 'COMMENT',
+    value: inner,
+    offset: start.offset,
+    line: start.line,
+    column: start.column
+  }
+}
+
+function tryReadRegexp (state, start) {
+  const s = state.source
+  let j = state.i + 1
+  let inClass = false
+  while (j < s.length) {
+    const c = s[j]
+    if (c === '\\') {
+      j += 2
+      continue
+    }
+    if (inClass) {
+      if (c === ']') inClass = false
+      j++
+      continue
+    }
+    if (c === '[') {
+      inClass = true
+      j++
+      continue
+    }
+    if (c === '\n') return null
+    if (c === '/') {
+      j++
+      while (j < s.length && /[gimsuy]/.test(s[j])) {
+        if (s[j] === 'm' && /[a-zA-Z]/.test(s[j + 1] || '')) break
+        j++
+      }
+      const raw = s.slice(state.i, j)
+      const n = j - state.i
+      advancePos(state, n)
+      return {
+        type: 'REGEXP',
+        value: raw,
+        offset: start.offset,
+        line: start.line,
+        column: start.column
+      }
+    }
+    j++
+  }
+  return null
+}
+
+function readString (state, quote, diagnostics) {
+  const start = pos(state)
+  let j = state.i + 1
+  const src = state.source
+  while (j < src.length) {
+    const c = src[j]
+    if (c === '\\') {
+      j += 2
+      continue
+    }
+    if (c === quote) {
+      const n = j + 1 - state.i
+      advancePos(state, n)
+      return {
+        type: 'STRING',
+        value: src.slice(start.offset, state.i),
+        offset: start.offset,
+        line: start.line,
+        column: start.column
+      }
+    }
+    if (c === '\n') break
+    j++
+  }
+  diagnostics.push({
+    code: 'E1001',
+    message: diag.e1001UnexpectedToken(quote),
+    offset: start.offset,
+    line: start.line,
+    column: start.column
+  })
+  state.i = src.length
+  return null
+}
+
+export function nextToken (state, ctx) {
+  skipWs(state)
+  if (state.i >= state.source.length) {
+    return { type: 'EOF', offset: state.i, line: state.line, column: state.col }
+  }
+
+  const s = state.source.slice(state.i)
+  const start = pos(state)
+
+  if (s.startsWith('--')) {
+    const ct = readCommentToken(state, ctx.diagnostics)
+    if (ct) return ct
+    return { type: 'EOF', offset: state.i, line: state.line, column: state.col }
+  }
+
+  if (s.startsWith('->')) {
+    advancePos(state, 2)
+    return { type: 'ARROW', value: '->', offset: start.offset, line: start.line, column: start.column }
+  }
+
+  const c0 = s[0]
+  if (c0 === '&') {
+    advancePos(state, 1)
+    return { type: 'AMP', value: '&', offset: start.offset, line: start.line, column: start.column }
+  }
+
+  if (c0 === '(' || c0 === ')' || c0 === '[' || c0 === ']' || c0 === '|' || c0 === ':' || c0 === ';') {
+    advancePos(state, 1)
+    return { type: c0, value: c0, offset: start.offset, line: start.line, column: start.column }
+  }
+
+  if (c0 === '"' || c0 === "'") {
+    const str = readString(state, c0, ctx.diagnostics)
+    if (str) return str
+    return { type: 'EOF', offset: state.i, line: state.line, column: state.col }
+  }
+
+  if (c0 === '/') {
+    if (ctx.regexpOk) {
+      const r = tryReadRegexp(state, start)
+      if (r) return r
+    }
+    const m = s.match(RE_WORD_REF)
+    if (m) {
+      advancePos(state, m[0].length)
+      return {
+        type: 'WORD_REF',
+        value: m[1],
+        offset: start.offset,
+        line: start.line,
+        column: start.column
+      }
+    }
+    ctx.diagnostics.push({
+      code: 'E1001',
+      message: diag.e1001UnexpectedToken('/'),
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    })
+    advancePos(state, 1)
+    return nextToken(state, ctx)
+  }
+
+  if (s.startsWith('nil') && !/[a-zA-Z0-9_]/.test(s[3] || '')) {
+    advancePos(state, 3)
+    return { type: 'NIL', value: 'nil', offset: start.offset, line: start.line, column: start.column }
+  }
+  if (s.startsWith('true') && !/[a-zA-Z0-9_]/.test(s[4] || '')) {
+    advancePos(state, 4)
+    return { type: 'TRUE', value: 'true', offset: start.offset, line: start.line, column: start.column }
+  }
+  if (s.startsWith('false') && !/[a-zA-Z0-9_]/.test(s[5] || '')) {
+    advancePos(state, 5)
+    return { type: 'FALSE', value: 'false', offset: start.offset, line: start.line, column: start.column }
+  }
+
+  let m = s.match(RE_BIGINT)
+  if (m) {
+    advancePos(state, m[0].length)
+    return { type: 'BIGINT', value: m[0], offset: start.offset, line: start.line, column: start.column }
+  }
+  m = s.match(RE_NUM)
+  if (m) {
+    advancePos(state, m[0].length)
+    return { type: 'NUMBER', value: m[0], offset: start.offset, line: start.line, column: start.column }
+  }
+
+  m = s.match(RE_EFFECT_ADD)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'EFFECT_ADD',
+      value: m[0],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+  m = s.match(RE_EFFECT_SUB)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'EFFECT_SUB',
+      value: m[0],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+
+  m = s.match(RE_MODULE)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'MODULE',
+      value: m[1],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+
+  m = s.match(RE_WORD_DEF)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'WORD_DEF',
+      value: m[1],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+
+  m = s.match(RE_MOD_WORD)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'MODULE_WORD_REF',
+      module: m[1],
+      word: m[2],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+  m = s.match(RE_MOD_TYPE)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'MODULE_TYPE_REF',
+      module: m[1],
+      typeName: m[2],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+  m = s.match(RE_STACK_VAR)
+  if (m) {
+    advancePos(state, m[0].length)
+    return {
+      type: 'STACK_VAR',
+      value: m[1],
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+
+  m = s.match(RE_IDENT)
+  if (m) {
+    const word = m[1]
+    advancePos(state, m[0].length)
+    if (/^[A-Z]/.test(word)) {
+      return {
+        type: 'TYPE_NAME',
+        value: word,
+        offset: start.offset,
+        line: start.line,
+        column: start.column
+      }
+    }
+    return {
+      type: 'LOWER_IDENT',
+      value: word,
+      offset: start.offset,
+      line: start.line,
+      column: start.column
+    }
+  }
+
+  ctx.diagnostics.push({
+    code: 'E1001',
+    message: diag.e1001UnexpectedToken(c0),
+    offset: start.offset,
+    line: start.line,
+    column: start.column
+  })
+  advancePos(state, 1)
+  return nextToken(state, ctx)
+}
+
+export function readPathToken (state, ctx) {
+  skipWs(state)
+  if (state.i >= state.source.length) return null
+  const start = pos(state)
+  const s = state.source.slice(state.i)
+  const m = s.match(RE_PATH)
+  if (!m || m[1] === '') return null
+  advancePos(state, m[1].length)
+  return {
+    type: 'PATH',
+    value: m[1],
+    offset: start.offset,
+    line: start.line,
+    column: start.column
+  }
+}
+
+export function createLexerState (source) {
+  return { source, i: 0, line: 1, col: 1 }
+}