liquor 0.1.1 → 0.9.1

data/lib/liquor/grammar/lexer.rl

@@ -0,0 +1,297 @@
+%%{
+
+machine liquor;
+
+action add_line_start {
+  line_starts.push(p + 1)
+}
+
+whitespace  = [\t ]+;
+symbol      = [a-zA-Z_];
+any_newline = '\n' @ add_line_start | any;
+identifier  = symbol ( symbol | digit )*;
+
+lblock  = '{%';
+rblock  = '%}';
+
+linterp = '{{';
+rinterp = '}}';
+
+lcomment = '{!';
+rcomment = '!}';
+
+action string_append {
+  string << data[p]
+}
+
+action string_end {
+  tok.(:string, string.dup, ts: str_start)
+  string.clear
+  fgoto code;
+}
+
+action runaway {
+  runaway = true
+}
+
+action error {
+  symbol = data[p]
+  if symbol == "\n"
+    symbol = "end of line"
+  else
+    symbol = "`#{symbol.inspect[1..-2]}'"
+  end
+
+  error = SyntaxError.new("unexpected #{symbol}",
+    file:  name,
+    line:  line_starts.count - 1,
+    start: p - line_starts.last,
+    end:   p - line_starts.last)
+  raise error
+}
+
+comment := |*
+    lcomment => { fcall comment; };
+    rcomment => { fret; };
+    any_newline;
+*|;
+
+dqstring := |*
+    '\\"'  => { string << '"' };
+    '\\\\' => { string << '\\' };
+    '"'    => string_end;
+    [^\n] @eof runaway
+           => string_append;
+    "\n"   => error;
+*|;
+
+sqstring := |*
+    "\\'"  => { string << "'" };
+    "\\\\" => { string << '\\' };
+    "'"    => string_end;
+    [^\n] @eof runaway
+           => string_append;
+    "\n"   => error;
+*|;
+
+integer := |*
+    digit+  => { tok.(:integer, data[ts...te].to_i) };
+    symbol  => error;
+    any     => { fhold; fgoto code; };
+*|;
+
+tag_start := |*
+    whitespace;
+
+    'end ' identifier =>
+      { tag = data[ts + 4...te]
+        if tag_stack.last == tag
+          fixtok.(:lblock2)
+          tok.(:endtag)
+          pop_tag.()
+        else
+          (sl, sc), (el, ec) = loc.(ts), loc.(te - 1)
+          info = { file: name, line: sl, start: sc, end: ec }
+          if tag_stack.any?
+            raise SyntaxError.new("unmatched `end #{tag}', expected `end #{tag_stack.last}'", info)
+          else
+            raise SyntaxError.new("unexpected `end #{tag}'", info)
+          end
+        end
+        fgoto code;
+      };
+
+    identifier ':' =>
+      { fixtok.(:lblock2)
+        tok.(:keyword, data[ts...te - 1])
+        fgoto code;
+      };
+
+    identifier =>
+      { tag = data[ts...te]
+
+        if tag_conts.include? tag
+          fixtok.(:lblock2)
+          tok.(:keyword, tag)
+          fgoto code;
+        end
+
+        tok.(:ident, tag)
+        last_tag = tag
+
+        fgoto code;
+      };
+
+    any =>
+      { fhold; fgoto code; };
+*|;
+
+code := |*
+    whitespace;
+
+    identifier => { tok.(:ident, data[ts...te]) };
+
+    digit => { fhold; fgoto integer; };
+
+    identifier %{ kw_stop = p } ':' whitespace* rblock =>
+      { tok.(:keyword, data[ts...kw_stop], te: kw_stop)
+        tok.(:rblock,  nil,                ts: te - 2)
+        push_last_tag.()
+        fgoto plaintext;
+      };
+
+    identifier ':' =>
+      { tok.(:keyword, data[ts...te-1]) };
+
+    '=' whitespace* rblock =>
+      {
+        tok.(:keyword, '=', te: ts + 1)
+        tok.(:rblock,  nil, ts: te - 2)
+        push_last_tag.()
+        fgoto plaintext;
+      };
+
+    '=' =>
+      { tok.(:keyword, '=') };
+
+    ','  => { tok.(:comma) };
+    '.'  => { tok.(:dot)   };
+
+    '['  => { tok.(:lbracket) };
+    ']'  => { tok.(:rbracket) };
+
+    '('  => { tok.(:lparen) };
+    ')'  => { tok.(:rparen) };
+
+    '|'  => { tok.(:pipe) };
+
+    '+'  => { tok.(:op_plus)  };
+    '-'  => { tok.(:op_minus) };
+    '*'  => { tok.(:op_mul)   };
+    '/'  => { tok.(:op_div)   };
+    '%'  => { tok.(:op_mod)   };
+
+    '==' => { tok.(:op_eq)  };
+    '!=' => { tok.(:op_neq) };
+    '>'  => { tok.(:op_gt)  };
+    '>=' => { tok.(:op_geq) };
+    '<'  => { tok.(:op_lt)  };
+    '<=' => { tok.(:op_leq) };
+
+    '!'  => { tok.(:op_not) };
+
+    '&&' => { tok.(:op_and) };
+    '||' => { tok.(:op_or) };
+
+    '"'  => { str_start = p; fgoto dqstring; };
+    "'"  => { str_start = p; fgoto sqstring; };
+
+    rinterp => { tok.(:rinterp);                 fgoto plaintext; };
+    rblock  => { tok.(:rblock);  last_tag = nil; fgoto plaintext; };
+
+    any => error;
+*|;
+
+plaintext := |*
+    ( '\\{' [%{!]? | '{' ( [^%{!] | '\n' @ add_line_start ) | any_newline - '{' )* =>
+      { tok.(:plaintext, data[ts...te]); };
+
+    '{{' =>
+      { tok.(:linterp); fgoto code; };
+
+    '{%' =>
+      { tok.(:lblock);  fgoto tag_start; };
+
+    '{!' =>
+      { fcall comment; };
+
+    any_newline =>
+      { fhold; fgoto code; };
+*|;
+
+}%%
+
+module Liquor
+  module Lexer
+    %% write data;
+
+    def self.lex(data, name='(code)', registered_tags={})
+      eof    = data.length
+      ts     = nil # token start
+      te     = nil # token end
+      stack  = []
+
+      # Strings
+      string    = ""
+      str_start = nil
+      runaway   = false
+
+      # Tags
+      kw_stop    = nil
+
+      line_starts = [0]
+
+      find_line_start = ->(index) {
+        line_starts.
+            each_index.find { |i|
+              line_starts[i + 1].nil? || line_starts[i + 1] > index
+            }
+      }
+      loc = ->(index) {
+        line_start_index = find_line_start.(index)
+        [ line_start_index, index - line_starts[line_start_index] ]
+      }
+
+      # Tag stack
+      tag_stack  = []
+      tag_conts  = []
+      last_tag   = nil
+
+      update_tag_cont = ->() {
+        tag = tag_stack.last
+        if registered_tags.include?(tag)
+          tag_conts = registered_tags[tag].continuations
+        end
+      }
+      push_last_tag = ->() {
+        if last_tag
+          tag_stack.push last_tag
+          last_tag = nil
+          update_tag_cont.()
+        end
+      }
+      pop_tag = ->() {
+        tag_stack.pop
+        update_tag_cont.()
+      }
+
+      tokens = []
+
+      fixtok = ->(new_type) {
+        tokens.last[0] = new_type
+      }
+      tok = ->(type, data=nil, options={}) {
+        sl, sc, el, ec = *loc.(options[:ts] || ts),
+                         *loc.(options[:te] || te - 1)
+        tokens << [type, { file: name, line: sl, start: sc, end: ec }, *data]
+      }
+
+      %% write init;
+      %% write exec;
+
+      if runaway
+        line_start_index = find_line_start.(str_start)
+        line_start = line_starts[line_start_index]
+
+        error = SyntaxError.new("literal not terminated",
+          file:  name,
+          line:  line_start_index,
+          start: str_start - line_start,
+          end:   str_start - line_start)
+        raise error
+      end
+
+      tokens
+    end
+  end
+end

data/lib/liquor/grammar/parser.racc

@@ -0,0 +1,288 @@
+class Liquor::Parser
+  token comma dot endtag ident integer keyword lblock lblock2 lbracket
+        linterp lparen op_div op_eq op_gt op_geq op_lt op_leq op_minus
+        op_mod op_mul op_neq op_not op_plus pipe plaintext rblock
+        rbracket rinterp rparen string tag_ident
+
+  prechigh
+    left dot
+    nonassoc op_uminus op_not
+    left op_mul op_div op_mod
+    left op_plus op_minus
+    left op_eq op_neq op_lt op_leq op_gt op_geq
+    left op_and
+    left op_or
+  preclow
+
+  expect 15
+
+  start block
+
+rule
+  block: /* empty */
+      { result = [] }
+    | plaintext block
+      { result = [ val[0], *val[1] ] }
+    | interp block
+      { result = [ val[0], *val[1] ] }
+    | tag block
+      { result = [ val[0], *val[1] ] }
+
+  interp:
+      linterp expr rinterp
+      { result = [ :interp, retag(val), val[1] ] }
+    | linterp filter_chain rinterp
+      { result = [ :interp, retag(val), val[1] ] }
+
+  primary_expr:
+      ident
+    | lparen expr rparen
+      { result = [ val[1][0], retag(val), *val[1][2..-1] ] }
+
+  expr:
+      integer
+    | string
+    | tuple
+    | ident function_args
+      { result = [ :call,   retag(val), val[0], val[1] ] }
+    | expr lbracket expr rbracket
+      { result = [ :index,  retag(val), val[0], val[2] ] }
+    | expr dot ident function_args
+      { result = [ :external, retag(val), val[0], val[2], val[3] ] }
+    | expr dot ident
+      { result = [ :external, retag(val), val[0], val[2], nil ] }
+    | op_minus expr =op_uminus
+      { result = [ :uminus, retag(val), val[1] ] }
+    | op_not expr
+      { result = [ :not, retag(val), val[1] ] }
+    | expr op_mul expr
+      { result = [ :mul, retag(val), val[0], val[2] ] }
+    | expr op_div expr
+      { result = [ :div, retag(val), val[0], val[2] ] }
+    | expr op_mod expr
+      { result = [ :mod, retag(val), val[0], val[2] ] }
+    | expr op_plus expr
+      { result = [ :plus, retag(val), val[0], val[2] ] }
+    | expr op_minus expr
+      { result = [ :minus, retag(val), val[0], val[2] ] }
+    | expr op_eq expr
+      { result = [ :eq, retag(val), val[0], val[2] ] }
+    | expr op_neq expr
+      { result = [ :neq, retag(val), val[0], val[2] ] }
+    | expr op_lt expr
+      { result = [ :lt, retag(val), val[0], val[2] ] }
+    | expr op_leq expr
+      { result = [ :leq, retag(val), val[0], val[2] ] }
+    | expr op_gt expr
+      { result = [ :gt, retag(val), val[0], val[2] ] }
+    | expr op_geq expr
+      { result = [ :geq, retag(val), val[0], val[2] ] }
+    | expr op_and expr
+      { result = [ :and, retag(val), val[0], val[2] ] }
+    | expr op_or expr
+      { result = [ :or, retag(val), val[0], val[2] ] }
+    | primary_expr
+
+  tuple:
+      lbracket tuple_content rbracket
+      { result = [ :tuple, retag(val), val[1].compact ] }
+
+  tuple_content:
+      expr comma tuple_content
+      { result = [ val[0], *val[2] ] }
+    | expr
+      { result = [ val[0] ] }
+    | /* empty */
+      { result = [ ] }
+
+  function_args:
+      lparen function_args_inside rparen
+      { result = [ :args, retag(val), *val[1] ] }
+
+  function_args_inside:
+      expr function_keywords
+      { result = [ val[0], val[1][2] ] }
+    | function_keywords
+      { result = [ nil,    val[0][2] ] }
+
+  function_keywords:
+      keyword expr function_keywords
+      { name = val[0][2].to_sym
+        tail = val[2][2]
+        loc  = retag([ val[0], val[1] ])
+
+        if tail.include? name
+          @errors << SyntaxError.new("duplicate keyword argument `#{val[0][2]}'",
+              tail[name][1])
+        end
+
+        hash = {
+          name => [ val[1][0], loc, *val[1][2..-1] ]
+        }.merge(tail)
+
+        result = [ :keywords, retag([ loc, val[2] ]), hash ]
+      }
+    | /* empty */
+      { result = [ :keywords, nil, {} ] }
+
+  filter_chain:
+      expr pipe filter_chain_cont
+      { result = [ val[0], *val[2] ].
+            reduce { |tree, node| node[3][2] = tree; node }
+      }
+
+  filter_chain_cont:
+      filter_call pipe filter_chain_cont
+      { result = [ val[0], *val[2] ] }
+    | filter_call
+      { result = [ val[0] ] }
+
+  filter_call:
+      ident function_keywords
+      { ident_loc = val[0][1]
+        empty_args_loc = { line:  ident_loc[:line],
+                           start: ident_loc[:end] + 1,
+                           end:   ident_loc[:end] + 1, }
+        result = [ :call, val[0][1], val[0],
+                   [ :args, val[1][1] || empty_args_loc, nil, val[1][2] ] ]
+      }
+
+  tag:
+      lblock ident expr tag_first_cont
+      { result = [ :tag, retag(val), val[1], val[2], *reduce_tag_args(val[3][2]) ] }
+    | lblock ident tag_first_cont
+      { result = [ :tag, retag(val), val[1], nil,    *reduce_tag_args(val[2][2]) ] }
+
+  # Racc cannot do lookahead across rules. I had to add states
+  # explicitly to avoid S/R conflicts. You are not expected to
+  # understand this.
+
+  tag_first_cont:
+      rblock
+      { result = [ :cont,  retag(val), [] ] }
+    | keyword tag_first_cont2
+      { result = [ :cont,  retag(val), [ val[0], *val[1][2] ] ] }
+
+  tag_first_cont2:
+      rblock block lblock2 tag_next_cont
+      { result = [ :cont2, val[0][1],  [ [:block, val[0][1], val[1] ], *val[3] ] ] }
+    | expr tag_first_cont
+      { result = [ :cont2, retag(val), [ val[0], *val[1][2] ] ] }
+
+  tag_next_cont:
+      endtag rblock
+      { result = [] }
+    | keyword tag_next_cont2
+      { result = [ val[0], *val[1] ] }
+
+  tag_next_cont2:
+      rblock block lblock2 tag_next_cont
+      { result = [ [:block, val[0][1], val[1] ], *val[3] ] }
+    | expr keyword tag_next_cont3
+      { result = [ val[0], val[1], *val[2] ] }
+
+  tag_next_cont3:
+      rblock block lblock2 tag_next_cont
+      { result = [ [:block, val[0][1], val[1] ], *val[3] ] }
+    | expr tag_next_cont
+      { result = [ val[0], *val[1] ] }
+
+---- inner
+  attr_reader :errors, :ast
+
+  def initialize(tags={})
+    super()
+
+    @errors = []
+    @ast    = nil
+    @tags   = tags
+  end
+
+  def success?
+    @errors.empty?
+  end
+
+  def parse(string, name='(code)')
+    @errors.clear
+    @name = name
+    @ast  = nil
+
+    begin
+      @stream = Lexer.lex(string, @name, @tags)
+      @ast = do_parse
+    rescue Liquor::SyntaxError => e
+      @errors << e
+    end
+
+    success?
+  end
+
+  def next_token
+    tok = @stream.shift
+    [ tok[0], tok ] if tok
+  end
+
+  TOKEN_NAME_MAP = {
+    :comma    => ',',
+    :dot      => '.',
+    :lblock   => '{%',
+    :rblock   => '%}',
+    :linterp  => '{{',
+    :rinterp  => '}}',
+    :lbracket => '[',
+    :rbracket => ']',
+    :lparen   => '(',
+    :rparen   => ')',
+    :pipe     => '|',
+    :op_not   => '!',
+    :op_mul   => '*',
+    :op_div   => '/',
+    :op_mod   => '%',
+    :op_plus  => '+',
+    :op_minus => '-',
+    :op_eq    => '==',
+    :op_neq   => '!=',
+    :op_lt    => '<',
+    :op_leq   => '<=',
+    :op_gt    => '>',
+    :op_geq   => '>=',
+    :keyword  => 'keyword argument name',
+    :kwarg    => 'keyword argument',
+    :ident    => 'identifier',
+  }
+
+  def on_error(error_token_id, error_token, value_stack)
+    if token_to_str(error_token_id) == "$end"
+      raise Liquor::SyntaxError.new("unexpected end of program", {
+        file: @name
+      })
+    else
+      type, (loc, value) = error_token
+      type = TOKEN_NAME_MAP[type] || type
+
+      raise Liquor::SyntaxError.new("unexpected token `#{type}'", loc)
+    end
+  end
+
+  def retag(nodes)
+    loc = nodes.map { |node| node[1] }.compact
+    first, *, last = loc
+    return first if last.nil?
+
+    {
+      file:  first[:file],
+      line:  first[:line],
+      start: first[:start],
+      end:    last[:end],
+    }
+  end
+
+  def reduce_tag_args(list)
+    list.each_slice(2).reduce([]) { |args, (k, v)|
+      if v[0] == :block
+        args << [ :blockarg, retag([ k, v ]), k, v[2] || [] ]
+      else
+        args << [ :kwarg,    retag([ k, v ]), k, v          ]
+      end
+    }
+  end