fop_lang 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ed95bb4708820a186e6485cc29dbb47286b0f309a1caf91af7778d768b0efb3
-  data.tar.gz: 85d41728ddae13f3667f0a2d55a5c4dcbc26e8217d4f31466e6ed92038859881
+  metadata.gz: 1166e1e43fd54ed2263db8a37ff288431a6152543869d961f007a134483f1a4b
+  data.tar.gz: 17e55b17448c38a37afb6e24a5798c87de368a352bdb0c74df13ad865a7e3ad0
 SHA512:
-  metadata.gz: e650bdf66d8d0b5dcb603eae494f38d4969a19647053b4e81e0612705f5b16a5755d007e4ce01fad9b487224d66eff462738f1e37b011ba2a4bf4a45b0203bb3
-  data.tar.gz: 99b31736236785cecc85b9bb23ccc3e366713cd23dd04c992a8fff6676a316d5741bfe5058e17cd0812c43d8bf1979aa7546184386018c1f92ed2462a85eb5fb
+  metadata.gz: cbf5d8c7f6c10ca395518cbd7bf0e9083b1f217b3523386a482f19e5cdf47a16aa7ffb50a6290f89f0ec80ba113900c499944fdf35b7e2e71a941d537483a7e1
+  data.tar.gz: bff3c613a575687d0d3223c5bd60bb1128b0ae78accf2e5101228c3ec14d61f46cdc798855af9d07692de0b25ba01737dbda97409d4e8f44881e9bbe6da9c523

data/lib/fop/compiler.rb

@@ -0,0 +1,72 @@
+require_relative 'parser'
+
+module Fop
+  module Compiler
+    def self.compile(src)
+      parser = Parser.new(src)
+      nodes, errors = parser.parse
+
+      instructions = nodes.map { |node|
+        case node
+        when Nodes::Text, Nodes::Regex
+          Instructions.regex_match(node.regex)
+        when Nodes::Expression
+          Instructions::ExpressionMatch.new(node)
+        else
+          raise "Unknown node type #{node}"
+        end
+      }
+
+      return nil, errors if errors.any?
+      return instructions, nil
+    end
+
+    module Instructions
+      BLANK = "".freeze
+      OPERATIONS = {
+        "=" => ->(_val, arg) { arg || BLANK },
+        "+" => ->(val, arg) { val.to_i + arg.to_i },
+        "-" => ->(val, arg) { val.to_i - arg.to_i },
+        ">" => ->(val, arg) { val + arg },
+        "<" => ->(val, arg) { arg + val },
+      }
+
+      def self.regex_match(regex)
+        ->(input) { input.slice! regex }
+      end
+
+      class ExpressionMatch
+        def initialize(node)
+          @regex = node.regex&.regex
+          @op = node.operator ? OPERATIONS.fetch(node.operator) : nil
+          @regex_match = node.regex_match
+          if node.arg&.any? { |a| a.is_a? Integer }
+            @arg, @arg_with_caps = nil, node.arg
+          else
+            @arg = node.arg&.join("")
+            @arg_with_caps = nil
+          end
+        end
+
+        def call(input)
+          if (match = @regex.match(input))
+            val = match.to_s
+            blank = val == BLANK
+            input.sub!(val, BLANK) unless blank
+            found_val = @regex_match || !blank
+            arg = @arg_with_caps ? sub_caps(@arg_with_caps, match.captures) : @arg
+            @op && found_val ? @op.call(val, arg) : val
+          end
+        end
+
+        private
+
+        def sub_caps(args, caps)
+          args.map { |a|
+            a.is_a?(Integer) ? caps[a].to_s : a
+          }.join("")
+        end
+      end
+    end
+  end
+end

data/lib/fop/nodes.rb

@@ -1,47 +1,31 @@
 module Fop
   module Nodes
-    Text = Struct.new(:wildcard, :str) do
-      def consume!(input)
-        @regex ||= Regexp.new((wildcard ? ".*" : "^") + Regexp.escape(str))
-        input.slice!(@regex)
-      end
-
+    Text = Struct.new(:wildcard, :str, :regex) do
       def to_s
         w = wildcard ? "*" : nil
-        "Text #{w}#{str}"
+        "[#{w}txt] #{str}"
       end
     end
 
-    Op = Struct.new(:wildcard, :match, :regex_match, :regex, :operator, :operator_arg, :operator_arg_w_caps, :expression) do
-      def consume!(input)
-        if (match = regex.match(input))
-          val = match.to_s
-          blank = val == Parser::BLANK
-          input.sub!(val, Parser::BLANK) unless blank
-          found_val = regex_match || !blank
-          arg = operator_arg_w_caps ? sub_caps(operator_arg_w_caps, match.captures) : operator_arg
-          expression && found_val ? expression.call(val, operator, arg) : val
-        end
+    Regex = Struct.new(:wildcard, :src, :regex) do
+      def to_s
+        w = wildcard ? "*" : nil
+        "[#{w}reg] #{src}"
       end
+    end
 
+    Expression = Struct.new(:wildcard, :match, :regex_match, :regex, :operator, :arg) do
       def to_s
         w = wildcard ? "*" : nil
-        s = "#{w}#{match}"
-        s << " #{operator} #{operator_arg}" if operator
+        s = "[#{w}exp] #{match}"
+        if operator
+          arg_str = arg
+            .map { |a| a.is_a?(Integer) ? "$#{a+1}" : a.to_s }
+            .join("")
+          s << " #{operator} #{arg_str}"
+        end
         s
       end
-
-      private
-
-      def sub_caps(tokens, caps)
-        tokens.map { |t|
-          case t
-          when String then t
-          when Parser::CaptureGroup then caps[t.index].to_s
-          else raise Parser::Error, "Unexpected #{t} in capture group"
-          end
-        }.join("")
-      end
     end
   end
 end

data/lib/fop/parser.rb

@@ -1,181 +1,162 @@
+require_relative 'tokenizer'
 require_relative 'nodes'
 
 module Fop
-  module Parser
-    Error = Class.new(StandardError)
-    CaptureGroup = Struct.new(:index)
+  class Parser
+    DIGIT = /^[0-9]$/
+    REGEX_START = "^".freeze
+    REGEX_LAZY_WILDCARD = ".*?".freeze
+    REGEX_MATCHES = {
+      "N" => "[0-9]+".freeze,
+      "W" => "\\w+".freeze,
+      "A" => "[a-zA-Z]+".freeze,
+      "*" => ".*".freeze,
+    }.freeze
+    OPS_WITH_OPTIONAL_ARGS = [Tokenizer::OP_REPLACE]
+    TR_REGEX = /.*/
+
+    Error = Struct.new(:type, :token, :message) do
+      def to_s
+        "#{type.to_s.capitalize} error: #{message} at column #{token.pos}"
+      end
+    end
 
-    MATCH_NUM = "N".freeze
-    MATCH_WORD = "W".freeze
-    MATCH_ALPHA = "A".freeze
-    MATCH_WILD = "*".freeze
-    BLANK = "".freeze
-    OP_REPLACE = "=".freeze
-    OP_APPEND = ">".freeze
-    OP_PREPEND = "<".freeze
-    OP_ADD = "+".freeze
-    OP_SUB = "-".freeze
-    OP_MUL = "*".freeze
-    OP_DIV = "/".freeze
-    VAR = "$".freeze
-    CAP_NUM = /^[1-9]$/
+    attr_reader :errors
 
-    EXP_REPLACE = ->(_val, _op, arg) { arg || BLANK }
-    EXP_MATH = ->(val, op, arg) { val.to_i.send(op, arg.to_i) }
-    EXP_APPEND = ->(val, _op, arg) { val + arg }
-    EXP_PREPEND = ->(val, _op, arg) { arg + val }
+    def initialize(src, debug: false)
+      @tokenizer = Tokenizer.new(src)
+      @errors = []
+    end
 
-    def self.parse!(tokens)
+    def parse
       nodes = []
-      curr_node = nil
-
-      tokens.each { |token|
-        case curr_node
-        when nil
-          curr_node = new_node token
-        when :wildcard
-          curr_node = new_node token, true
-          raise Error, "Unexpected * after wildcard" if curr_node == :wildcard
-        when Nodes::Text
-          curr_node, finished_node = parse_text curr_node, token
-          nodes << finished_node if finished_node
-        when Nodes::Op
-          nodes << curr_node
-          curr_node = new_node token
+      wildcard = false
+      eof = false
+      # Top-level parsing. It will always be looking for a String, Regex, or Expression.
+      until eof
+        @tokenizer.reset_escapes!
+        t = @tokenizer.next
+        case t.type
+        when Tokens::WILDCARD
+          errors << Error.new(:syntax, t, "Consecutive wildcards") if wildcard
+          wildcard = true
+        when Tokens::TEXT
+          reg = build_regex!(wildcard, t, Regexp.escape(t.val))
+          nodes << Nodes::Text.new(wildcard, t.val, reg)
+          wildcard = false
+        when Tokens::EXP_OPEN
+          nodes << parse_exp!(wildcard)
+          wildcard = false
+        when Tokens::REG_DELIM
+          nodes << parse_regex!(wildcard)
+          wildcard = false
+        when Tokens::EOF
+          eof = true
         else
-          raise Error, "Unexpected node #{curr_node}"
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}")
         end
-      }
-
-      case curr_node
-      when nil
-        # noop
-      when :wildcard
-        nodes << Nodes::Text.new(true, "")
-      when Nodes::Text, Nodes::Op
-        nodes << curr_node
-      else
-        raise Error, "Unexpected end node #{curr_node}"
       end
-
-      nodes
+      nodes << Nodes::Text.new(true, "", TR_REGEX) if wildcard
+      return nodes, @errors
     end
 
-    private
+    def parse_exp!(wildcard = false)
+      exp = Nodes::Expression.new(wildcard)
+      parse_exp_match! exp
+      op_token = parse_exp_operator! exp
+      if exp.operator
+        parse_exp_arg! exp, op_token
+      end
+      return exp
+    end
 
-    def self.new_node(token, wildcard = false)
-      case token
-      when Tokenizer::Char
-        Nodes::Text.new(wildcard, token.char.clone)
-      when Tokenizer::Op
-        op = Nodes::Op.new(wildcard)
-        parse_op! op, token
-        op
-      when :wildcard
-        :wildcard
+    def parse_exp_match!(exp)
+      @tokenizer.escape.operators = false
+      t = @tokenizer.next
+      case t.type
+      when Tokens::TEXT, Tokens::WILDCARD
+        exp.match = t.val
+        if (src = REGEX_MATCHES[exp.match])
+          reg = Regexp.new((exp.wildcard ? REGEX_LAZY_WILDCARD : REGEX_START) + src)
+          exp.regex = Nodes::Regex.new(exp.wildcard, src, reg)
+        else
+          errors << Error.new(:name, t, "Unknown match type '#{exp.match}'") if exp.regex.nil?
+        end
+      when Tokens::REG_DELIM
+        exp.regex = parse_regex!(exp.wildcard)
+        exp.match = exp.regex&.src
+        exp.regex_match = true
+        @tokenizer.reset_escapes!
       else
-        raise Error, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string or a regex")
       end
     end
 
-    # @return current node
-    # @return finished node
-    def self.parse_text(node, token)
-      case token
-      when Tokenizer::Char
-        node.str << token.char
-        return node, nil
-      when Tokenizer::Op
-        op = new_node token
-        return op, node
-      when :wildcard
-        return :wildcard, node
+    def parse_exp_operator!(exp)
+      @tokenizer.escape.operators = false
+      t = @tokenizer.next
+      case t.type
+      when Tokens::EXP_CLOSE
+        # no op
+      when Tokens::OPERATOR
+        exp.operator = t.val
       else
-        raise Error, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected an operator")
       end
+      t
     end
 
-    def self.parse_op!(node, token)
-      # parse the matching type
-      node.regex =
-        case token.match
-        when Tokenizer::Char
-          node.match = token.match.char
-          node.regex_match = false
-          case node.match
-          when MATCH_NUM then Regexp.new((node.wildcard ? ".*?" : "^") + "[0-9]+")
-          when MATCH_WORD then Regexp.new((node.wildcard ? ".*?" : "^") + "\\w+")
-          when MATCH_ALPHA then Regexp.new((node.wildcard ? ".*?" : "^") + "[a-zA-Z]+")
-          when MATCH_WILD then /.*/
-          else raise Error, "Unknown match type '#{node.match}'"
-          end 
-        when Tokenizer::Regex
-          node.match = "/#{token.match.src}/"
-          node.regex_match = true
-          Regexp.new((node.wildcard ? ".*?" : "^") + token.match.src)
-        when nil
-          raise Error, "Empty operation"
+    def parse_exp_arg!(exp, op_token)
+      @tokenizer.escape.operators = true
+      @tokenizer.escape.regex = true
+      @tokenizer.escape.regex_capture = false if exp.regex_match
+
+      exp.arg = []
+      found_close, eof = false, false
+      until found_close or eof
+        t = @tokenizer.next
+        case t.type
+        when Tokens::TEXT
+          exp.arg << t.val
+        when Tokens::REG_CAPTURE
+          exp.arg << t.val.to_i - 1
+          errors << Error.new(:syntax, t, "Invalid regex capture; must be between 0 and 9 (found #{t.val})") unless t.val =~ DIGIT
+          errors << Error.new(:syntax, t, "Unexpected regex capture; expected str or '}'") if !exp.regex_match
+        when Tokens::EXP_CLOSE
+          found_close = true
+        when Tokens::EOF
+          eof = true
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
         else
-          raise Error, "Unexpected #{token.match}"
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
         end
+      end
 
-      # parse the operator (if any)
-      if token.operator
-        raise Error, "Unexpected #{token.operator} for operator" unless token.operator.is_a? Tokenizer::Char
-        node.operator = token.operator.char
-        node.operator_arg = token.arg if token.arg and token.arg != BLANK
-        node.operator_arg_w_caps = parse_captures! node.operator_arg if node.operator_arg and node.regex_match
-        node.expression =
-          case node.operator
-          when OP_REPLACE
-            EXP_REPLACE
-          when OP_ADD, OP_SUB, OP_MUL, OP_DIV
-            raise Error, "Operator #{node.operator} is only available for numeric matches" unless node.match == MATCH_NUM
-            raise Error, "Operator #{node.operator} expects an argument" if node.operator_arg.nil?
-            EXP_MATH
-          when OP_APPEND
-            raise Error, "Operator #{node.operator} expects an argument" if node.operator_arg.nil?
-            EXP_APPEND
-          when OP_PREPEND
-            raise Error, "Operator #{node.operator} expects an argument" if node.operator_arg.nil?
-            EXP_PREPEND
-          else
-            raise Error, "Unknown operator #{node.operator}"
-          end
+      if exp.arg.size != 1 and !OPS_WITH_OPTIONAL_ARGS.include?(exp.operator)
+        errors << Error.new(:arg, op_token, "Operator '#{op_token.val}' requires an argument")
       end
     end
 
-    def self.parse_captures!(arg)
-      i = 0
-      iend = arg.size - 1
-      escape = false
-      nodes = []
-
-      until i > iend
-        char = arg[i]
-        i += 1
-
-        if escape
-          nodes << char
-          escape = false
-          next
-        end
-
-        case char
-        when Tokenizer::ESCAPE
-          escape = true
-        when VAR
-          num = arg[i].to_s
-          raise Error, "Capture group number must be between 1 and 9; found '#{num}'" unless num =~ CAP_NUM
-          nodes << CaptureGroup.new(num.to_i - 1)
-          i += 1
-        else
-          nodes << char
-        end
+    def parse_regex!(wildcard)
+      @tokenizer.regex_mode!
+      t = @tokenizer.next
+      reg = Nodes::Regex.new(wildcard, t.val)
+      if t.type == Tokens::TEXT
+        reg.regex = build_regex!(wildcard, t)
+      else
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string of regex")
       end
 
-      raise Error, "Trailing escape" if escape
-      nodes
+      t = @tokenizer.next
+      errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string of regex") unless t.type == Tokens::REG_DELIM
+      reg
+    end
+
+    def build_regex!(wildcard, token, src = token.val)
+      Regexp.new((wildcard ? REGEX_LAZY_WILDCARD : REGEX_START) + src)
+    rescue RegexpError => e
+      errors << Error.new(:regex, token, e.message)
+      nil
     end
   end
 end

data/lib/fop/program.rb

@@ -1,22 +1,16 @@
-require_relative 'tokenizer'
-require_relative 'parser'
-
 module Fop
   class Program
-    attr_reader :nodes
-
-    def initialize(src)
-      tokens = Tokenizer.new(src).tokenize!
-      @nodes = Parser.parse! tokens
+    def initialize(instructions)
+      @instructions = instructions
     end
 
     def apply(input)
       input = input.clone
       output =
-        @nodes.reduce("") { |acc, token|
-          section = token.consume!(input)
-          return nil if section.nil?
-          acc + section.to_s
+        @instructions.reduce("") { |acc, ins|
+          result = ins.call(input)
+          return nil if result.nil?
+          acc + result.to_s
         }
       input.empty? ? output : nil
     end

data/lib/fop/tokenizer.rb

@@ -1,144 +1,175 @@
+require_relative 'tokens'
+
 module Fop
   class Tokenizer
-    Char = Struct.new(:char)
-    Op = Struct.new(:match, :operator, :arg)
-    Regex = Struct.new(:src)
-    Error = Class.new(StandardError)
+    Token = Struct.new(:pos, :type, :val)
+    Error = Struct.new(:pos, :message)
+    Escapes = Struct.new(:operators, :regex_capture, :regex, :regex_escape, :wildcards, :exp)
 
-    OP_OPEN = "{".freeze
-    OP_CLOSE = "}".freeze
+    EXP_OPEN = "{".freeze
+    EXP_CLOSE = "}".freeze
     ESCAPE = "\\".freeze
     WILDCARD = "*".freeze
-    REGEX_MARKER = "/".freeze
+    REGEX_DELIM = "/".freeze
+    REGEX_CAPTURE = "$".freeze
+    OP_REPLACE = "=".freeze
+    OP_APPEND = ">".freeze
+    OP_PREPEND = "<".freeze
+    OP_ADD = "+".freeze
+    OP_SUB = "-".freeze
+
+    #
+    # Controls which "mode" the tokenizer is currently in. This is a necessary result of the syntax lacking
+    # explicit string delimiters. That *could* be worked around by requiring users to escape all reserved chars,
+    # but that's ugly af. Instead, the parser continually assesses the current context and flips these flags on
+    # or off to auto-escape certain chars for the next token.
+    #
+    attr_reader :escape
 
     def initialize(src)
       @src = src
       @end = src.size - 1
+      @start_i = 0
+      @i = 0
+      reset_escapes!
     end
 
-    def tokenize!
-      tokens = []
-      escape = false
-      i = 0
-      until i > @end do
-        char = @src[i]
-        i += 1
-
-        if escape
-          tokens << Char.new(char)
-          escape = false
-          next
-        end
-
-        case char
-        when ESCAPE
-          escape = true
-        when OP_OPEN
-          i, op = operation! i
-          tokens << op
-        when OP_CLOSE
-          raise "Unexpected #{OP_CLOSE}"
-        when WILDCARD
-          tokens << :wildcard
-        else
-          tokens << Char.new(char)
-        end
-      end
-
-      raise Error, "Trailing escape" if escape
-      tokens
+    # Auto-escape operators and regex capture vars. Appropriate for top-level syntax.
+    def reset_escapes!
+      @escape = Escapes.new(true, true)
     end
 
-    private
-
-    def operation!(i)
-      found_close = false
-      op = Op.new(nil, nil, "")
+    # Auto-escape anything you'd find in a regular expression
+    def regex_mode!
+      @escape.regex = false # look for the final /
+      @escape.regex_escape = true # pass \ through to the regex engine UNLESS it's followed by a /
+      @escape.wildcards = true
+      @escape.operators = true
+      @escape.regex_capture = true
+      @escape.exp = true
+    end
 
-      # Find matcher
-      until found_close or op.match or i > @end do
-        char = @src[i]
-        i += 1
-        case char
-        when OP_CLOSE
-          found_close = true
-        when REGEX_MARKER
-          i, reg = regex! i
-          op.match = reg
+    def next
+      return Token.new(@i, Tokens::EOF) if @i > @end
+      char = @src[@i]
+      case char
+      when EXP_OPEN
+        @i += 1
+        token! Tokens::EXP_OPEN
+      when EXP_CLOSE
+        @i += 1
+        token! Tokens::EXP_CLOSE
+      when WILDCARD
+        @i += 1
+        token! Tokens::WILDCARD, WILDCARD
+      when REGEX_DELIM
+        if @escape.regex
+          get_str!
         else
-          op.match = Char.new(char)
+          @i += 1
+          token! Tokens::REG_DELIM
         end
-      end
-
-      # Find operator
-      until found_close or op.operator or i > @end do
-        char = @src[i]
-        i += 1
-        case char
-        when OP_CLOSE
-          found_close = true
+      when REGEX_CAPTURE
+        if @escape.regex_capture
+          get_str!
         else
-          op.operator = Char.new(char)
-        end
-      end
-
-      # Find operator arg
-      escape = false
-      until found_close or i > @end do
-        char = @src[i]
-        i += 1
-
-        if escape
-          op.arg << char
-          escape = false
-          next
+          @i += 1
+          t = token! Tokens::REG_CAPTURE, @src[@i]
+          @i += 1
+          @start_i = @i
+          t
         end
-
-        case char
-        when ESCAPE
-          escape = true
-        when OP_OPEN
-          raise "Unexpected #{OP_OPEN}"
-        when OP_CLOSE
-          found_close = true
+      when OP_REPLACE, OP_APPEND, OP_PREPEND, OP_ADD, OP_SUB
+        if @escape.operators
+          get_str!
         else
-          op.arg << char
+          @i += 1
+          token! Tokens::OPERATOR, char
         end
+      else
+        get_str!
       end
-
-      raise Error, "Unclosed operation" if !found_close
-      raise Error, "Trailing escape" if escape
-      return i, op
     end
 
-    def regex!(i)
-      escape = false
-      found_close = false
-      src = ""
+    private
+
+    def token!(type, val = nil)
+      t = Token.new(@start_i, type, val)
+      @start_i = @i
+      t
+    end
 
-      until found_close or i > @end
-        char = @src[i]
-        i += 1
+    def get_str!
+      str = ""
+      escape, found_end = false, false
+      until found_end or @i > @end
+        char = @src[@i]
 
         if escape
-          src << char
+          @i += 1
+          str << char
           escape = false
           next
         end
 
         case char
         when ESCAPE
-          escape = true
-        when REGEX_MARKER
-          found_close = true
+          @i += 1
+          if @escape.regex_escape and @src[@i] != REGEX_DELIM
+            str << char
+          else
+            escape = true
+          end
+        when EXP_OPEN
+          if @escape.exp
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when EXP_CLOSE
+          if @escape.exp
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when WILDCARD
+          if @escape.wildcards
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when REGEX_DELIM
+          if @escape.regex
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when REGEX_CAPTURE
+          if @escape.regex_capture
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when OP_REPLACE, OP_APPEND, OP_PREPEND, OP_ADD, OP_SUB
+          if @escape.operators
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
         else
-          src << char
+          @i += 1
+          str << char
         end
       end
 
-      raise Error, "Unclosed regex" if !found_close
-      raise Error, "Trailing escape" if escape
-      return i, Regex.new(src)
+      return Token.new(@i - 1, Tokens::TR_ESC) if escape
+      token! Tokens::TEXT, str
     end
   end
 end

data/lib/fop/tokens.rb

@@ -0,0 +1,13 @@
+module Fop
+  module Tokens
+    TEXT = :TXT
+    EXP_OPEN = :"{"
+    EXP_CLOSE = :"}"
+    REG_CAPTURE = :"$"
+    REG_DELIM = :/
+    WILDCARD = :*
+    OPERATOR = :op
+    TR_ESC = :"trailing escape"
+    EOF = :EOF
+  end
+end

data/lib/fop/version.rb

@@ -1,3 +1,3 @@
 module Fop
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/fop_lang.rb

@@ -1,12 +1,22 @@
 require_relative 'fop/version'
+require_relative 'fop/compiler'
 require_relative 'fop/program'
 
 def Fop(src)
-  ::Fop::Program.new(src)
+  ::Fop.compile!(src)
 end
 
 module Fop
+  def self.compile!(src)
+    prog, errors = compile(src)
+    # TODO better exception
+    raise "Fop errors: " + errors.map(&:message).join(",") if errors
+    prog
+  end
+
   def self.compile(src)
-    Program.new(src)
+    instructions, errors = ::Fop::Compiler.compile(src)
+    return nil, errors if errors
+    return Program.new(instructions), nil
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fop_lang
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jordan Hollinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-16 00:00:00.000000000 Z
+date: 2021-08-20 00:00:00.000000000 Z
 dependencies: []
 description: A micro expression language for Filter and OPerations on text
 email: jordan.hollinger@gmail.com
@@ -17,10 +17,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - README.md
+- lib/fop/compiler.rb
 - lib/fop/nodes.rb
 - lib/fop/parser.rb
 - lib/fop/program.rb
 - lib/fop/tokenizer.rb
+- lib/fop/tokens.rb
 - lib/fop/version.rb
 - lib/fop_lang.rb
 homepage: https://jhollinger.github.io/fop-lang-rb/