@o3co/ts.hocon 0.1.0

package/src/internal/lexer/lexer.ts

@@ -0,0 +1,168 @@
+import { ParseError } from '../../errors.js'
+import type { Token, TokenKind } from './token.js'
+
+export function tokenize(input: string): Token[] {
+  if (input.charCodeAt(0) === 0xfeff) input = input.slice(1)
+
+  const tokens: Token[] = []
+  let pos = 0
+  let line = 1
+  let col = 1
+  let hadSpace = false
+
+  function peek(offset = 0): string { return input[pos + offset] ?? '' }
+
+  function advance(): string {
+    const ch = input[pos++] ?? ''
+    if (ch === '\n') { line++; col = 1 } else { col++ }
+    return ch
+  }
+
+  function push(kind: TokenKind, value: string, l: number, c: number, isQuoted = false): void {
+    tokens.push({ kind, value, line: l, col: c, isQuoted, precedingSpace: hadSpace })
+    hadSpace = false
+  }
+
+  while (pos < input.length) {
+    const sl = line, sc = col
+    const ch = peek()
+
+    // Whitespace (not newline)
+    if (ch === ' ' || ch === '\t' || ch === '\r') {
+      advance(); hadSpace = true; continue
+    }
+
+    // Newline
+    if (ch === '\n') {
+      advance()
+      if (tokens[tokens.length - 1]?.kind !== 'newline') {
+        push('newline', '\n', sl, sc)
+      }
+      continue
+    }
+
+    // Comments
+    if (ch === '/' && peek(1) === '/') {
+      while (pos < input.length && peek() !== '\n') advance()
+      hadSpace = true; continue
+    }
+    if (ch === '#') {
+      while (pos < input.length && peek() !== '\n') advance()
+      hadSpace = true; continue
+    }
+
+    // Single-char punctuation
+    const single: Record<string, TokenKind> = { '{': 'lbrace', '}': 'rbrace', '[': 'lbracket', ']': 'rbracket', ',': 'comma', ':': 'colon' }
+    if (ch in single) { advance(); push(single[ch] as TokenKind, ch, sl, sc); continue }
+
+    // = and +=
+    if (ch === '=') { advance(); push('equals', '=', sl, sc); continue }
+    if (ch === '+' && peek(1) === '=') { advance(); advance(); push('plus_equals', '+=', sl, sc); continue }
+
+    // Substitution ${...} or ${?...}
+    if (ch === '$' && peek(1) === '{') {
+      advance(); advance()
+      const optional = peek() === '?'
+      if (optional) advance()
+      let path = ''
+      while (pos < input.length && peek() !== '}') {
+        if (peek() === '\n') throw new ParseError('unterminated substitution', sl, sc)
+        path += advance()
+      }
+      if (peek() !== '}') throw new ParseError('unterminated substitution', sl, sc)
+      advance()
+      push(optional ? 'opt_subst' : 'subst', path.trim(), sl, sc)
+      continue
+    }
+
+    // Triple-quoted string
+    if (ch === '"' && peek(1) === '"' && peek(2) === '"') {
+      advance(); advance(); advance()
+      let value = ''
+      while (pos < input.length) {
+        if (peek() === '"') {
+          // Count consecutive quotes
+          let quoteCount = 0
+          while (pos < input.length && peek() === '"') {
+            quoteCount++
+            advance()
+          }
+          if (quoteCount >= 3) {
+            // Last 3 quotes are the closing delimiter; extras are content
+            for (let i = 0; i < quoteCount - 3; i++) value += '"'
+            break
+          }
+          // Fewer than 3 quotes — they are content
+          for (let i = 0; i < quoteCount; i++) value += '"'
+          continue
+        }
+        value += advance()
+      }
+      if (value.startsWith('\n')) value = value.slice(1)
+      push('triple_string', value, sl, sc, true)
+      continue
+    }
+
+    // Quoted string
+    if (ch === '"') {
+      advance()
+      let value = ''
+      while (pos < input.length && peek() !== '"') {
+        if (peek() === '\n') throw new ParseError('unterminated string', sl, sc)
+        if (peek() === '\\') {
+          advance()
+          const esc = advance()
+          switch (esc) {
+            case 'n': value += '\n'; break
+            case 't': value += '\t'; break
+            case 'r': value += '\r'; break
+            case '"': value += '"'; break
+            case '\\': value += '\\'; break
+            case '/': value += '/'; break
+            case 'b': value += '\b'; break
+            case 'f': value += '\f'; break
+            case 'u': {
+              const hex = input.slice(pos, pos + 4)
+              value += String.fromCharCode(parseInt(hex, 16))
+              for (let i = 0; i < 4; i++) advance()
+              break
+            }
+            default: value += esc
+          }
+        } else {
+          value += advance()
+        }
+      }
+      if (peek() !== '"') throw new ParseError('unterminated string', sl, sc)
+      advance()
+      push('string', value, sl, sc, true)
+      continue
+    }
+
+    // Unquoted string (stops at terminators and $)
+    if (isUnquotedStart(ch)) {
+      let value = ''
+      while (pos < input.length && isUnquotedContinue(peek(), () => peek(1))) {
+        value += advance()
+      }
+      push('unquoted', value.trimEnd(), sl, sc)
+      continue
+    }
+
+    throw new ParseError(`unexpected character: ${JSON.stringify(ch)}`, sl, sc)
+  }
+
+  tokens.push({ kind: 'eof', value: '', line, col, isQuoted: false, precedingSpace: false })
+  return tokens
+}
+
+function isUnquotedStart(ch: string): boolean {
+  return ch !== '' && !'{}[],:=+#\n\r\t "$'.includes(ch)
+}
+
+function isUnquotedContinue(ch: string, nextFn: () => string): boolean {
+  if (ch === '' || '{}[],:=\n\r\t #"$'.includes(ch) || ch === ' ') return false
+  if (ch === '+' && nextFn() === '=') return false
+  if (ch === '/' && nextFn() === '/') return false
+  return true
+}

package/src/internal/lexer/token.ts

@@ -0,0 +1,20 @@
+export type TokenKind =
+  | 'lbrace' | 'rbrace'
+  | 'lbracket' | 'rbracket'
+  | 'comma' | 'colon' | 'equals' | 'plus_equals'
+  | 'newline'
+  | 'string'         // "..." quoted string
+  | 'triple_string'  // """..."""
+  | 'unquoted'       // bare word, number, true/false/null
+  | 'subst'          // ${path}
+  | 'opt_subst'      // ${?path}
+  | 'eof'
+
+export type Token = {
+  kind: TokenKind
+  value: string           // always string; parser converts to number/bool/null
+  line: number
+  col: number
+  isQuoted: boolean       // true for "..." and """..."""
+  precedingSpace: boolean // true if preceded by whitespace (concat detection)
+}

package/src/internal/parser/ast.ts

@@ -0,0 +1,17 @@
+export type Pos = { line: number; col: number; file?: string }
+
+export type AstNode =
+  | { kind: 'object'; fields: AstField[]; pos: Pos }
+  | { kind: 'array'; items: AstNode[]; pos: Pos }
+  | { kind: 'scalar'; value: string | number | boolean | null; pos: Pos }
+  | { kind: 'concat'; nodes: AstNode[]; pos: Pos }
+  | { kind: 'subst'; path: string; optional: boolean; pos: Pos }
+  | { kind: 'include'; path: string; pos: Pos }
+
+// key が空配列のとき include ディレクティブを表す（value は include ノード）
+export type AstField = {
+  key: string[]     // 各要素はドット分割済み。quoted キーはドット分割しない
+  value: AstNode
+  append: boolean   // true なら += 演算子
+  pos: Pos
+}

package/src/internal/parser/parser.ts

@@ -0,0 +1,229 @@
+import { ParseError } from '../../errors.js'
+import type { Token } from '../lexer/token.js'
+import type { AstNode, AstField, Pos } from './ast.js'
+
+export function parseTokens(tokens: Token[]): AstNode {
+  let pos = 0
+
+  function peek(): Token { return tokens[pos] ?? { kind: 'eof', value: '', line: 0, col: 0, isQuoted: false, precedingSpace: false } }
+  function advance(): Token {
+    const t = tokens[pos]
+    if (pos < tokens.length) pos++
+    return t ?? { kind: 'eof', value: '', line: 0, col: 0, isQuoted: false, precedingSpace: false }
+  }
+  function skip(...kinds: string[]): void {
+    while (kinds.includes(peek().kind)) advance()
+  }
+
+  function currentPos(): Pos {
+    const t = peek()
+    return { line: t.line, col: t.col }
+  }
+
+  function parseObject(expectClosingBrace: boolean): AstNode {
+    const p = currentPos()
+    const fields: AstField[] = []
+
+    while (true) {
+      skip('newline')
+      const t = peek()
+      if (t.kind === 'eof' || t.kind === 'rbrace') break
+
+      // include directive
+      if (t.kind === 'unquoted' && t.value === 'include') {
+        advance()
+        fields.push(parseInclude())
+        // trailing separator after include
+        skip('newline')
+        if (peek().kind === 'comma') advance()
+        skip('newline')
+        continue
+      }
+
+      // key
+      const keyPos = currentPos()
+      const key = parseKey()
+
+      // value separator (optional)
+      skip('newline')
+      let append = false
+      const sep = peek()
+      if (sep.kind === 'equals') advance()
+      else if (sep.kind === 'plus_equals') { advance(); append = true }
+      else if (sep.kind === 'colon') advance()
+      else if (sep.kind === 'lbrace') { /* key { ... } shorthand — no advance, value parser handles it */ }
+      else if (sep.kind !== 'newline' && sep.kind !== 'eof') {
+        throw new ParseError(`unexpected token after key: ${sep.kind}`, sep.line, sep.col)
+      }
+
+      skip('newline')
+      const value = parseValue()
+      fields.push({ key, value, append, pos: keyPos })
+
+      // trailing separator
+      skip('newline')
+      if (peek().kind === 'comma') advance()
+      skip('newline')
+    }
+
+    if (expectClosingBrace) {
+      const t = peek()
+      if (t.kind !== 'rbrace') throw new ParseError('expected }', t.line, t.col)
+      advance()
+    }
+
+    return { kind: 'object', fields, pos: p }
+  }
+
+  function parseKey(): string[] {
+    const segments: string[] = []
+    let trailingDot = false
+    while (true) {
+      const t = peek()
+      if (t.kind === 'string') {
+        advance()
+        segments.push(t.value) // quoted: no dot split
+        trailingDot = false
+      } else if (t.kind === 'unquoted') {
+        advance()
+        // Split unquoted key at dots
+        const parts = t.value.split('.')
+        const filtered = parts.filter(s => s.length > 0)
+        segments.push(...filtered)
+        // If the unquoted value ended with a dot, the next token continues the key
+        trailingDot = t.value.endsWith('.')
+      } else {
+        if (segments.length === 0) throw new ParseError(`expected key, got ${t.kind}`, t.line, t.col)
+        break
+      }
+
+      // If the last unquoted segment ended with a dot, continue to next token
+      if (trailingDot) continue
+
+      // Check for explicit dot separator between segments (e.g. "a"."b")
+      // A lone dot as an unquoted token with no preceding space continues the key
+      const next = peek()
+      if (next.kind === 'unquoted' && next.value === '.' && !next.precedingSpace) {
+        advance() // consume the dot separator
+        trailingDot = true
+        continue
+      }
+
+      break
+    }
+    return segments
+  }
+
+  function parseInclude(): AstField {
+    const p = currentPos()
+    skip('newline')
+    const t = peek()
+    let path: string
+
+    if (t.kind === 'string') {
+      // include "path"
+      path = advance().value
+    } else if (t.kind === 'unquoted' && (t.value === 'file(' || t.value === 'file')) {
+      // include file("path")
+      advance()
+      // Skip tokens until we find the quoted path string
+      while (peek().kind !== 'string' && peek().kind !== 'eof') advance()
+      if (peek().kind === 'eof') throw new ParseError('expected include path', t.line, t.col)
+      path = advance().value
+      // Skip closing ) and anything else on this line
+      while (peek().kind !== 'newline' && peek().kind !== 'rbrace' && peek().kind !== 'eof') advance()
+    } else {
+      throw new ParseError(`expected include path, got ${t.kind}`, t.line, t.col)
+    }
+
+    return {
+      key: [],
+      value: { kind: 'include', path, pos: p },
+      append: false,
+      pos: p,
+    }
+  }
+
+  function parseValue(): AstNode {
+    const p = currentPos()
+    const parts: AstNode[] = []
+
+    while (true) {
+      const t = peek()
+      if (t.kind === 'eof' || t.kind === 'newline' || t.kind === 'rbrace' || t.kind === 'rbracket' || t.kind === 'comma') break
+
+      // If there was whitespace before this token and we already have parts,
+      // insert a space node for proper string concatenation.
+      const hadSpace = t.precedingSpace && parts.length > 0
+
+      let node: AstNode
+      if (t.kind === 'lbrace') {
+        advance()
+        node = parseObject(true)
+      } else if (t.kind === 'lbracket') {
+        advance()
+        node = parseArray()
+      } else if (t.kind === 'subst' || t.kind === 'opt_subst') {
+        advance()
+        node = { kind: 'subst', path: t.value, optional: t.kind === 'opt_subst', pos: { line: t.line, col: t.col } }
+      } else if (t.kind === 'string' || t.kind === 'triple_string') {
+        advance()
+        node = { kind: 'scalar', value: t.value, pos: { line: t.line, col: t.col } }
+      } else if (t.kind === 'unquoted') {
+        advance()
+        node = { kind: 'scalar', value: parseScalarValue(t.value), pos: { line: t.line, col: t.col } }
+      } else if ((t.kind === 'colon' || t.kind === 'equals') && parts.length > 0) {
+        // In value concat context, colon/equals after at least one part are plain string chars
+        // e.g.  url = ${host}:/path  or  x = ${a}=b
+        advance()
+        node = { kind: 'scalar', value: t.value, pos: { line: t.line, col: t.col } }
+      } else {
+        break
+      }
+      if (hadSpace) {
+        parts.push({ kind: 'scalar', value: ' ', pos: { line: t.line, col: t.col } })
+      }
+      parts.push(node)
+    }
+
+    if (parts.length === 0) throw new ParseError('expected value', peek().line, peek().col)
+    if (parts.length === 1) return parts[0]!
+    return { kind: 'concat', nodes: parts, pos: p }
+  }
+
+  function parseArray(): AstNode {
+    const p = currentPos()
+    const items: AstNode[] = []
+
+    while (true) {
+      skip('newline')
+      if (peek().kind === 'rbracket' || peek().kind === 'eof') break
+      items.push(parseValue())
+      skip('newline')
+      if (peek().kind === 'comma') advance()
+      skip('newline')
+    }
+
+    const t = peek()
+    if (t.kind !== 'rbracket') throw new ParseError('expected ]', t.line, t.col)
+    advance()
+    return { kind: 'array', items, pos: p }
+  }
+
+  function parseScalarValue(raw: string): string | number | boolean | null {
+    if (raw === 'true') return true
+    if (raw === 'false') return false
+    if (raw === 'null') return null
+    const n = Number(raw)
+    if (!isNaN(n) && raw.trim() !== '') return n
+    return raw
+  }
+
+  skip('newline')
+  const t = peek()
+  if (t.kind === 'lbrace') {
+    advance()
+    return parseObject(true)
+  }
+  return parseObject(false)
+}