@starpak/lang 0.0.0-stage → 0.1.2

package/src/index.ts

@@ -0,0 +1,21 @@
+/**
+ * Spak Lang - Public API
+ */
+
+export * from './token'
+export * from './ast'
+export * from './lexer'
+export * from './parser'
+export * from './checker'
+export { CODES, error, warning, countBySeverity, formatDiagnostic } from './diagnostics'
+export type { Diagnostic, DiagnosticCode, Severity } from './diagnostics'
+export { emitTypeScript, typeAnn } from './emit/ts'
+export type { EmitOptions } from './emit/ts'
+export { emitLua } from './emit/lua'
+export type { EmitLuaOptions } from './emit/lua'
+export {
+  ANY, NIL, NUMBER, STRING, BOOLEAN,
+  fromTypeAnn, union, isNil, withoutNil, isAssignable,
+  print as printType,
+} from './types'
+export type { Type, AnyType, NilType, PrimType } from './types'

package/src/lexer.ts

@@ -0,0 +1,301 @@
+/**
+ * Spak Lang - Lexer
+ *
+ * Converts source text into a flat token stream. Newlines are significant
+ * (statement separators) and consecutive blank lines are collapsed into a
+ * single NEWLINE token.
+ */
+
+import { Token, TokenType } from './token'
+
+const KEYWORDS = new Set([
+  'and', 'break', 'class', 'const', 'continue', 'do', 'else', 'elseif',
+  'end', 'export', 'false', 'fn', 'for', 'from', 'global', 'if', 'import',
+  'in', 'match', 'nil', 'not', 'or', 'repeat', 'return', 'spawn', 'then',
+  'throw', 'true', 'try', 'catch', 'finally', 'type', 'unless', 'until',
+  'when', 'while', 'async', 'await', 'extends', 'self', 'as', 'default',
+])
+
+/** Longest match wins, so multi-character operators come first. */
+const OPERATORS = [
+  '...', '..<', '..', '|>', '?.', '??', '->', '==', '~=', '<=', '>=', '//',
+  '+', '-', '*', '/', '%', '^', '#', '<', '>', '=', '(', ')', '{', '}', '[',
+  ']', ',', ':', '.', '@', '|', '?',
+]
+
+const ESCAPES: Record<string, string> = {
+  n: '\n',
+  t: '\t',
+  r: '\r',
+  '0': '\0',
+  '\\': '\\',
+  '"': '"',
+  "'": "'",
+}
+
+function isDigit(ch: string | undefined): boolean {
+  return ch !== undefined && ch >= '0' && ch <= '9'
+}
+
+function isHexDigit(ch: string | undefined): boolean {
+  return ch !== undefined && (
+    (ch >= '0' && ch <= '9') ||
+    (ch >= 'a' && ch <= 'f') ||
+    (ch >= 'A' && ch <= 'F')
+  )
+}
+
+function isIdentStart(ch: string | undefined): boolean {
+  return ch !== undefined && (
+    (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_'
+  )
+}
+
+function isIdentPart(ch: string | undefined): boolean {
+  return isIdentStart(ch) || isDigit(ch)
+}
+
+function syntaxError(message: string, line: number, column: number): SyntaxError {
+  return new SyntaxError(`${message} (line ${line}, column ${column})`)
+}
+
+function decodeEscapes(raw: string): string {
+  return raw.replace(/\\(.)/g, (match, ch: string) => ESCAPES[ch] ?? ch)
+}
+
+export function tokenize(source: string): Token[] {
+  const tokens: Token[] = []
+  let index = 0
+  let line = 1
+  let column = 1
+
+  /** Pending `---` doc lines, attached to the next non-newline token. */
+  let pendingDoc: string[] | undefined
+  let pendingDocLine = 0
+
+  function advance(count = 1): string {
+    let consumed = ''
+    for (let i = 0; i < count && index < source.length; i++) {
+      const ch = source[index++]
+      consumed += ch
+      if (ch === '\n') {
+        line++
+        column = 1
+      } else {
+        column++
+      }
+    }
+    return consumed
+  }
+
+  function push(
+    type: TokenType,
+    value: string,
+    start: number,
+    startLine: number,
+    startColumn: number,
+    fstring?: boolean,
+  ): void {
+    const token: Token = {
+      type,
+      value,
+      line: startLine,
+      column: startColumn,
+      start,
+      end: index,
+    }
+    if (fstring) token.fstring = true
+    if (type !== TokenType.Newline && pendingDoc) {
+      token.doc = pendingDoc.join('\n')
+      pendingDoc = undefined
+    }
+    tokens.push(token)
+  }
+
+  function pushNewline(start: number, startLine: number, startColumn: number): void {
+    if (!tokens.length) return
+    if (tokens[tokens.length - 1].type === TokenType.Newline) return
+    push(TokenType.Newline, '\n', start, startLine, startColumn)
+  }
+
+  function scanNumber(): void {
+    const start = index
+    const startLine = line
+    const startColumn = column
+
+    if (source[index] === '0' && (source[index + 1] === 'x' || source[index + 1] === 'X')) {
+      advance(2)
+      let digits = 0
+      while (isHexDigit(source[index]) || source[index] === '_') {
+        if (source[index] !== '_') digits++
+        advance()
+      }
+      if (!digits) throw syntaxError('Invalid hexadecimal number', startLine, startColumn)
+    } else if (source[index] === '0' && (source[index + 1] === 'b' || source[index + 1] === 'B')) {
+      advance(2)
+      let digits = 0
+      while (source[index] === '0' || source[index] === '1' || source[index] === '_') {
+        if (source[index] !== '_') digits++
+        advance()
+      }
+      if (!digits) throw syntaxError('Invalid binary number', startLine, startColumn)
+    } else {
+      let digits = 0
+      while (isDigit(source[index]) || source[index] === '_') {
+        if (source[index] !== '_') digits++
+        advance()
+      }
+      if (source[index] === '.' && source[index + 1] !== '.') {
+        if (isDigit(source[index + 1]) || source[index + 1] === undefined) {
+          advance()
+          while (isDigit(source[index]) || source[index] === '_') {
+            if (source[index] !== '_') digits++
+            advance()
+          }
+        }
+      }
+      if (source[index] === 'e' || source[index] === 'E') {
+        let lookahead = index + 1
+        if (source[lookahead] === '+' || source[lookahead] === '-') lookahead++
+        if (isDigit(source[lookahead])) {
+          advance()
+          if (source[index] === '+' || source[index] === '-') advance()
+          while (isDigit(source[index]) || source[index] === '_') advance()
+        }
+      }
+      if (!digits) throw syntaxError('Invalid number', startLine, startColumn)
+    }
+
+    push(TokenType.Number, source.slice(start, index), start, startLine, startColumn)
+  }
+
+  function scanString(quote: string, fstring: boolean): void {
+    const start = index
+    const startLine = line
+    const startColumn = column
+    const multiline = source.startsWith('"""', index)
+    advance(multiline ? 3 : 1)
+    const closing = multiline ? '"""' : quote
+    let raw = ''
+
+    while (index < source.length) {
+      if (source.startsWith(closing, index)) {
+        advance(closing.length)
+        const value = fstring || multiline ? raw : decodeEscapes(raw)
+        push(fstring ? TokenType.FString : TokenType.String, value, start, startLine, startColumn, fstring)
+        return
+      }
+      const ch = source[index]
+      if (ch === '\\' && !multiline) {
+        raw += ch
+        advance()
+        if (index < source.length) {
+          raw += source[index]
+          advance()
+        }
+        continue
+      }
+      raw += ch
+      advance()
+    }
+
+    throw syntaxError('Unterminated string literal', startLine, startColumn)
+  }
+
+  while (index < source.length) {
+    const ch = source[index]
+    const start = index
+    const startLine = line
+    const startColumn = column
+
+    if (ch === ' ' || ch === '\t' || ch === '\r') {
+      advance()
+      continue
+    }
+
+    if (ch === '\n') {
+      // A blank line breaks a doc comment run.
+      if (pendingDoc && pendingDocLine !== startLine) pendingDoc = undefined
+      advance()
+      pushNewline(start, startLine, startColumn)
+      continue
+    }
+
+    if (ch === '-' && source[index + 1] === '-') {
+      // Block comment: --[[ ... ]]
+      if (source[index + 2] === '[' && source[index + 3] === '[') {
+        pendingDoc = undefined
+        advance(4)
+        while (index < source.length && !(source[index] === ']' && source[index + 1] === ']')) {
+          advance()
+        }
+        if (index >= source.length) {
+          throw syntaxError('Unterminated block comment', startLine, startColumn)
+        }
+        advance(2)
+        continue
+      }
+      // Line comment, including doc comments (--- ...)
+      const isDoc = source[index + 2] === '-'
+      while (index < source.length && source[index] !== '\n') advance()
+      if (isDoc) {
+        if (pendingDoc && pendingDocLine !== startLine - 1) pendingDoc = undefined
+        if (!pendingDoc) pendingDoc = []
+        pendingDoc.push(source.slice(start, index).replace(/^---\s?/, '').trimEnd())
+        pendingDocLine = startLine
+      } else {
+        pendingDoc = undefined
+      }
+      continue
+    }
+
+    if (ch === ';') {
+      advance()
+      push(TokenType.Semicolon, ';', start, startLine, startColumn)
+      continue
+    }
+
+    if (ch === '"' || ch === "'") {
+      scanString(ch, false)
+      continue
+    }
+
+    if (ch === 'f' && (source[index + 1] === '"' || source[index + 1] === "'")) {
+      advance()
+      scanString(source[index], true)
+      continue
+    }
+
+    if (isDigit(ch) || (ch === '.' && isDigit(source[index + 1]))) {
+      scanNumber()
+      continue
+    }
+
+    if (isIdentStart(ch)) {
+      while (isIdentPart(source[index])) advance()
+      const value = source.slice(start, index)
+      push(KEYWORDS.has(value) ? TokenType.Keyword : TokenType.Identifier, value, start, startLine, startColumn)
+      continue
+    }
+
+    const operator = OPERATORS.find((candidate) => source.startsWith(candidate, index))
+    if (operator) {
+      advance(operator.length)
+      push(TokenType.Operator, operator, start, startLine, startColumn)
+      continue
+    }
+
+    throw syntaxError(`Unexpected character "${ch}"`, line, column)
+  }
+
+  tokens.push({
+    type: TokenType.EOF,
+    value: '',
+    line,
+    column,
+    start: index,
+    end: index,
+  })
+
+  return tokens
+}