fop_lang 0.1.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62ded9f974ca670a6eca5d4b293f98195af25ab476e9b3571e7def5299499fa9
-  data.tar.gz: b53b6e276ee31f122571a096dd534ed5c1407d27384fab78d216886322751fcb
+  metadata.gz: 1166e1e43fd54ed2263db8a37ff288431a6152543869d961f007a134483f1a4b
+  data.tar.gz: 17e55b17448c38a37afb6e24a5798c87de368a352bdb0c74df13ad865a7e3ad0
 SHA512:
-  metadata.gz: 45509b09122e4f76f1f8219d8c0f094bc5ac77231dec25f6e0e9818932b68186d74c71dd74db80638dcfc9bae210892b563e660004cfd32c25f439c761c3edc2
-  data.tar.gz: 49c6153cf728e909e60bc8fa1f1214dcdd8f9532703c8bc3455999aac1aa421c629370e1c7669bc9a846b5886fa2f10b7f5b698426cae2830e73ca9623a32a4a
+  metadata.gz: cbf5d8c7f6c10ca395518cbd7bf0e9083b1f217b3523386a482f19e5cdf47a16aa7ffb50a6290f89f0ec80ba113900c499944fdf35b7e2e71a941d537483a7e1
+  data.tar.gz: bff3c613a575687d0d3223c5bd60bb1128b0ae78accf2e5101228c3ec14d61f46cdc798855af9d07692de0b25ba01737dbda97409d4e8f44881e9bbe6da9c523

data/README.md

@@ -1,44 +1,92 @@
 # fop_lang
 
-Fop is a tiny expression language implemented in Ruby for text filtering and modification.
+Fop (Filter and OPerations language) is an experimental, tiny expression language in the vein of awk and sed. This is a Ruby implementation. It is useful for simultaneously matching and transforming text input.
 
-## Examples
+```ruby
+gem 'fop_lang'
+```
+
+## Release Number Example
+
+This example takes in GitHub branch names, decides if they're release branches, and if so, increments the version number.
 
 ```ruby
-  f = Fop("release-{N}.{N+1}.{N=0}")
+  f = Fop('release-{N}.{N+1}.{N=0}')
 
-  puts f.apply("release-5.99.1")
-  =>           "release-5.100.0"
+  puts f.apply('release-5.99.1')
+  =>           'release-5.100.0'
 
-  puts f.apply("release-5")
+  puts f.apply('release-5')
   => nil
   # doesn't match the pattern
 ```
 
+## Anatomy of a Fop expression
+
+`Text Literal {Operation}`
+
+The above expression contains the only two parts of Fop (except for the wildcard and escape characters).
+
+**Text Literals**
+
+A text literal works how it sounds: the input must match it exactly. If it matches it passes through unchanged. The only exception is the `*` (wildcard) character, which matches 0 or more of anything. Wildcards can be used anywhere except inside `{...}` (operations).
+
+If `\` (escape) is used before the special characters `*`, `{` or `}`, then that character is treated like a text literal. It's recommended to use single-quoted Ruby strings with Fop expressions that so you don't need to double-escape.
+
+**Operations**
+
+Operations are the interesting part of Fop, and are specified between `{` and `}`. An Operation can consist of one to three parts:
+
+1. Matching class (required): Defines what characters the operation will match and operate on.
+  * `N` is the numeric class and will match one or more digits.
+  * `A` is the alpha class and will match one or more letters (lower or upper case).
+  * `W` is the word class and matches alphanumeric chars and underscores.
+  * `*` is the wildcard class and greedily matches everything after it.
+  * `/.../` matches on the supplied regex between the `/`'s. If you're regex contains a `/`, it must be escaped. Capture groups may be referenced in the operator argument as `$1`, `$2`, etc.
+3. Operator (optional): What to do to the matching characters.
+  * `=` Replace the matching character(s) with the given argument. If no argument is given, drop the matching chars.
+  * `>` Append the following chars to the matching value.
+  * `<` Prepend the following chars to the matching value.
+  * `+` Perform addition on the matching number and the argument (`N` only).
+  * `-` Subtract the argument from the matching number (`N` only).
+5. Operator argument (required for some operators): meaning varies by operator.
+
+## More Examples
+
 ```ruby
-  f = Fop("release-{N=5}.{N+1}.{N=0}")
+  f = Fop('release-{N=5}.{N+1}.{N=0}')
+
+  puts f.apply('release-4.99.1')
+  =>           'release-5.100.0'
+```
+
+```ruby
+  f = Fop('rel{/(ease)?/}-{N=5}.{N+1}.{N=0}')
+
+  puts f.apply('release-4.99.1')
+  =>           'release-5.100.0'
 
-  puts f.apply("release-4.99.1")
-  =>           "release-5.100.0"
+  puts f.apply('rel-4.99.1')
+  =>           'rel-5.100.0'
 ```
 
 ```ruby
-  f = Fop("release-*{N=5}.{N+100}.{N=0}")
+  f = Fop('release-*{N=5}.{N+100}.{N=0}')
 
-  puts f.apply("release-foo-4.100.1")
-  =>           "release-foo-5.200.0"
+  puts f.apply('release-foo-4.100.1')
+  =>           'release-foo-5.200.0'
 ```
 
 ```ruby
-  f = Fop("release-{N=5}.{N+1}.{N=0}{*=}")
+  f = Fop('release-{N=5}.{N+1}.{N=0}{*=}')
 
-  puts f.apply("release-4.100.1.foo.bar")
-  =>           "release-5.101.0"
+  puts f.apply('release-4.100.1.foo.bar')
+  =>           'release-5.101.0'
 ```
 
 ```ruby
-  f = Fop("{W=version}-{N=5}.{N+1}.{N=0}")
+  f = Fop('{W=version}-{N=5}.{N+1}.{N=0}')
 
-  puts f.apply("release-4.100.1")
-  =>           "version-5.101.0"
+  puts f.apply('release-4.100.1')
+  =>           'version-5.101.0'
 ```

data/lib/fop/compiler.rb

@@ -0,0 +1,72 @@
+require_relative 'parser'
+
+module Fop
+  module Compiler
+    def self.compile(src)
+      parser = Parser.new(src)
+      nodes, errors = parser.parse
+
+      instructions = nodes.map { |node|
+        case node
+        when Nodes::Text, Nodes::Regex
+          Instructions.regex_match(node.regex)
+        when Nodes::Expression
+          Instructions::ExpressionMatch.new(node)
+        else
+          raise "Unknown node type #{node}"
+        end
+      }
+
+      return nil, errors if errors.any?
+      return instructions, nil
+    end
+
+    module Instructions
+      BLANK = "".freeze
+      OPERATIONS = {
+        "=" => ->(_val, arg) { arg || BLANK },
+        "+" => ->(val, arg) { val.to_i + arg.to_i },
+        "-" => ->(val, arg) { val.to_i - arg.to_i },
+        ">" => ->(val, arg) { val + arg },
+        "<" => ->(val, arg) { arg + val },
+      }
+
+      def self.regex_match(regex)
+        ->(input) { input.slice! regex }
+      end
+
+      class ExpressionMatch
+        def initialize(node)
+          @regex = node.regex&.regex
+          @op = node.operator ? OPERATIONS.fetch(node.operator) : nil
+          @regex_match = node.regex_match
+          if node.arg&.any? { |a| a.is_a? Integer }
+            @arg, @arg_with_caps = nil, node.arg
+          else
+            @arg = node.arg&.join("")
+            @arg_with_caps = nil
+          end
+        end
+
+        def call(input)
+          if (match = @regex.match(input))
+            val = match.to_s
+            blank = val == BLANK
+            input.sub!(val, BLANK) unless blank
+            found_val = @regex_match || !blank
+            arg = @arg_with_caps ? sub_caps(@arg_with_caps, match.captures) : @arg
+            @op && found_val ? @op.call(val, arg) : val
+          end
+        end
+
+        private
+
+        def sub_caps(args, caps)
+          args.map { |a|
+            a.is_a?(Integer) ? caps[a].to_s : a
+          }.join("")
+        end
+      end
+    end
+  end
+end

data/lib/fop/nodes.rb

@@ -1,68 +1,30 @@
 module Fop
   module Nodes
-    Text = Struct.new(:wildcard, :str) do
-      def consume!(input)
-        @regex ||= Regexp.new((wildcard ? ".*" : "^") + Regexp.escape(str))
-        input.slice!(@regex)
-      end
-
+    Text = Struct.new(:wildcard, :str, :regex) do
       def to_s
         w = wildcard ? "*" : nil
-        "Text #{w}#{str}"
+        "[#{w}txt] #{str}"
       end
     end
 
-    Match = Struct.new(:wildcard, :tokens) do
-      NUM = "N".freeze
-      WORD = "W".freeze
-      WILD = "*".freeze
-      BLANK = "".freeze
-
-      def consume!(input)
-        if (val = input.slice!(@regex))
-          @expression && val != BLANK ? @expression.call(val) : val
-        end
-      end
-
+    Regex = Struct.new(:wildcard, :src, :regex) do
       def to_s
         w = wildcard ? "*" : nil
-        @op ? "#{w}#{@match} #{@op} #{@arg}" : "#{w}#{@match}"
+        "[#{w}reg] #{src}"
       end
+    end
 
-      def parse!
-        match = tokens.shift || raise(ParserError, "Empty match")
-        raise ParserError, "Unexpected #{match}" unless match.is_a? Tokenizer::Char
-
-        @match = match.char
-        @regex =
-          case @match
-          when NUM then Regexp.new((wildcard ? ".*?" : "^") + "[0-9]+")
-          when WORD then Regexp.new((wildcard ? ".*?" : "^") + "[a-zA-Z]+")
-          when WILD then /.*/
-          else raise ParserError, "Unknown match type '#{@match}'"
-          end
-
-        if (op = tokens.shift)
-          raise ParserError, "Unexpected #{op}" unless op.is_a? Tokenizer::Char
-          arg = tokens.reduce("") { |acc, t|
-            raise ParserError, "Unexpected #{t}" unless t.is_a? Tokenizer::Char
-            acc + t.char
-          }
-
-          @op = op.char
-          @arg = arg == BLANK ? nil : arg
-          @expression =
-            case @op
-            when "=" then ->(_) { @arg || BLANK }
-            when "+", "-", "*", "/"
-              raise ParserError, "Operator #{@op} is only available for numeric matches" unless @match == NUM
-              raise ParserError, "Operator #{@op} expects an argument" if @arg.nil?
-              ->(x) { x.to_i.send(@op, @arg.to_i) }
-            else raise ParserError, "Unknown operator #{@op}"
-            end
-        else
-          @op, @arg, @expression = nil, nil, nil
+    Expression = Struct.new(:wildcard, :match, :regex_match, :regex, :operator, :arg) do
+      def to_s
+        w = wildcard ? "*" : nil
+        s = "[#{w}exp] #{match}"
+        if operator
+          arg_str = arg
+            .map { |a| a.is_a?(Integer) ? "$#{a+1}" : a.to_s }
+            .join("")
+          s << " #{operator} #{arg_str}"
         end
+        s
       end
     end
   end

data/lib/fop/parser.rb

@@ -1,93 +1,162 @@
+require_relative 'tokenizer'
 require_relative 'nodes'
 
 module Fop
-  module Parser
-    Error = Class.new(StandardError)
+  class Parser
+    DIGIT = /^[0-9]$/
+    REGEX_START = "^".freeze
+    REGEX_LAZY_WILDCARD = ".*?".freeze
+    REGEX_MATCHES = {
+      "N" => "[0-9]+".freeze,
+      "W" => "\\w+".freeze,
+      "A" => "[a-zA-Z]+".freeze,
+      "*" => ".*".freeze,
+    }.freeze
+    OPS_WITH_OPTIONAL_ARGS = [Tokenizer::OP_REPLACE]
+    TR_REGEX = /.*/
 
-    def self.parse!(tokens)
-      stack = []
-      current_el = nil
+    Error = Struct.new(:type, :token, :message) do
+      def to_s
+        "#{type.to_s.capitalize} error: #{message} at column #{token.pos}"
+      end
+    end
+
+    attr_reader :errors
 
-      tokens.each { |token|
-        case current_el
-        when nil
-          current_el = new_element token
-        when :wildcard
-          current_el = new_element token, true
-          raise Error, "Unexpected * after wildcard" if current_el == :wildcard
-        when Nodes::Text
-          current_el = parse_text stack, current_el, token
-        when Nodes::Match
-          current_el = parse_match stack, current_el, token
+    def initialize(src, debug: false)
+      @tokenizer = Tokenizer.new(src)
+      @errors = []
+    end
+
+    def parse
+      nodes = []
+      wildcard = false
+      eof = false
+      # Top-level parsing. It will always be looking for a String, Regex, or Expression.
+      until eof
+        @tokenizer.reset_escapes!
+        t = @tokenizer.next
+        case t.type
+        when Tokens::WILDCARD
+          errors << Error.new(:syntax, t, "Consecutive wildcards") if wildcard
+          wildcard = true
+        when Tokens::TEXT
+          reg = build_regex!(wildcard, t, Regexp.escape(t.val))
+          nodes << Nodes::Text.new(wildcard, t.val, reg)
+          wildcard = false
+        when Tokens::EXP_OPEN
+          nodes << parse_exp!(wildcard)
+          wildcard = false
+        when Tokens::REG_DELIM
+          nodes << parse_regex!(wildcard)
+          wildcard = false
+        when Tokens::EOF
+          eof = true
         else
-          raise Error, "Unexpected token #{token} in #{current_el}"
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}")
         end
-      }
-
-      case current_el
-      when nil
-        # noop
-      when :wildcard
-        stack << Nodes::Text.new(true, "")
-      when Nodes::Text
-        stack << current_el
-      when Nodes::Match
-        raise Error, "Unclosed match"
       end
-
-      stack
+      nodes << Nodes::Text.new(true, "", TR_REGEX) if wildcard
+      return nodes, @errors
     end
 
-    private
+    def parse_exp!(wildcard = false)
+      exp = Nodes::Expression.new(wildcard)
+      parse_exp_match! exp
+      op_token = parse_exp_operator! exp
+      if exp.operator
+        parse_exp_arg! exp, op_token
+      end
+      return exp
+    end
 
-    def self.new_element(token, wildcard = false)
-      case token
-      when Tokenizer::Char
-        Nodes::Text.new(wildcard, token.char.clone)
-      when :match_open
-        Nodes::Match.new(wildcard, [])
-      when :match_close
-        raise ParserError, "Unmatched }"
-      when :wildcard
-        :wildcard
+    def parse_exp_match!(exp)
+      @tokenizer.escape.operators = false
+      t = @tokenizer.next
+      case t.type
+      when Tokens::TEXT, Tokens::WILDCARD
+        exp.match = t.val
+        if (src = REGEX_MATCHES[exp.match])
+          reg = Regexp.new((exp.wildcard ? REGEX_LAZY_WILDCARD : REGEX_START) + src)
+          exp.regex = Nodes::Regex.new(exp.wildcard, src, reg)
+        else
+          errors << Error.new(:name, t, "Unknown match type '#{exp.match}'") if exp.regex.nil?
+        end
+      when Tokens::REG_DELIM
+        exp.regex = parse_regex!(exp.wildcard)
+        exp.match = exp.regex&.src
+        exp.regex_match = true
+        @tokenizer.reset_escapes!
       else
-        raise ParserError, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string or a regex")
       end
     end
 
-    def self.parse_text(stack, text_el, token)
-      case token
-      when :match_open
-        stack << text_el
-        Nodes::Match.new(false, [])
-      when :match_close
-        raise ParserError.new, "Unexpected }"
-      when Tokenizer::Char
-        text_el.str << token.char
-        text_el
-      when :wildcard
-        stack << text_el
-        :wildcard
+    def parse_exp_operator!(exp)
+      @tokenizer.escape.operators = false
+      t = @tokenizer.next
+      case t.type
+      when Tokens::EXP_CLOSE
+        # no op
+      when Tokens::OPERATOR
+        exp.operator = t.val
       else
-        raise ParserError, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected an operator")
       end
+      t
     end
 
-    def self.parse_match(stack, match_el, token)
-      case token
-      when Tokenizer::Char
-        match_el.tokens << token
-        match_el
-      when :wildcard
-        match_el.tokens << Tokenizer::Char.new("*").freeze
-        match_el
-      when :match_close
-        match_el.parse!
-        stack << match_el
-        nil
+    def parse_exp_arg!(exp, op_token)
+      @tokenizer.escape.operators = true
+      @tokenizer.escape.regex = true
+      @tokenizer.escape.regex_capture = false if exp.regex_match
+
+      exp.arg = []
+      found_close, eof = false, false
+      until found_close or eof
+        t = @tokenizer.next
+        case t.type
+        when Tokens::TEXT
+          exp.arg << t.val
+        when Tokens::REG_CAPTURE
+          exp.arg << t.val.to_i - 1
+          errors << Error.new(:syntax, t, "Invalid regex capture; must be between 0 and 9 (found #{t.val})") unless t.val =~ DIGIT
+          errors << Error.new(:syntax, t, "Unexpected regex capture; expected str or '}'") if !exp.regex_match
+        when Tokens::EXP_CLOSE
+          found_close = true
+        when Tokens::EOF
+          eof = true
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
+        else
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
+        end
+      end
+
+      if exp.arg.size != 1 and !OPS_WITH_OPTIONAL_ARGS.include?(exp.operator)
+        errors << Error.new(:arg, op_token, "Operator '#{op_token.val}' requires an argument")
+      end
+    end
+
+    def parse_regex!(wildcard)
+      @tokenizer.regex_mode!
+      t = @tokenizer.next
+      reg = Nodes::Regex.new(wildcard, t.val)
+      if t.type == Tokens::TEXT
+        reg.regex = build_regex!(wildcard, t)
       else
-        raise ParserError, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string of regex")
       end
+
+      t = @tokenizer.next
+      errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string of regex") unless t.type == Tokens::REG_DELIM
+      reg
+    end
+
+    def build_regex!(wildcard, token, src = token.val)
+      Regexp.new((wildcard ? REGEX_LAZY_WILDCARD : REGEX_START) + src)
+    rescue RegexpError => e
+      errors << Error.new(:regex, token, e.message)
+      nil
     end
   end
 end

data/lib/fop/program.rb

@@ -1,22 +1,16 @@
-require_relative 'tokenizer'
-require_relative 'parser'
-
 module Fop
   class Program
-    attr_reader :nodes
-
-    def initialize(src)
-      tokens = Tokenizer.tokenize! src
-      @nodes = Parser.parse! tokens
+    def initialize(instructions)
+      @instructions = instructions
     end
 
     def apply(input)
       input = input.clone
       output =
-        @nodes.reduce("") { |acc, token|
-          section = token.consume!(input)
-          return nil if section.nil?
-          acc + section.to_s
+        @instructions.reduce("") { |acc, ins|
+          result = ins.call(input)
+          return nil if result.nil?
+          acc + result.to_s
         }
       input.empty? ? output : nil
     end

data/lib/fop/tokenizer.rb

@@ -1,34 +1,175 @@
+require_relative 'tokens'
+
 module Fop
-  module Tokenizer
-    Char = Struct.new(:char)
-    Error = Class.new(StandardError)
-
-    def self.tokenize!(src)
-      tokens = []
-      escape = false
-      src.each_char { |char|
+  class Tokenizer
+    Token = Struct.new(:pos, :type, :val)
+    Error = Struct.new(:pos, :message)
+    Escapes = Struct.new(:operators, :regex_capture, :regex, :regex_escape, :wildcards, :exp)
+
+    EXP_OPEN = "{".freeze
+    EXP_CLOSE = "}".freeze
+    ESCAPE = "\\".freeze
+    WILDCARD = "*".freeze
+    REGEX_DELIM = "/".freeze
+    REGEX_CAPTURE = "$".freeze
+    OP_REPLACE = "=".freeze
+    OP_APPEND = ">".freeze
+    OP_PREPEND = "<".freeze
+    OP_ADD = "+".freeze
+    OP_SUB = "-".freeze
+
+    #
+    # Controls which "mode" the tokenizer is currently in. This is a necessary result of the syntax lacking
+    # explicit string delimiters. That *could* be worked around by requiring users to escape all reserved chars,
+    # but that's ugly af. Instead, the parser continually assesses the current context and flips these flags on
+    # or off to auto-escape certain chars for the next token.
+    #
+    attr_reader :escape
+
+    def initialize(src)
+      @src = src
+      @end = src.size - 1
+      @start_i = 0
+      @i = 0
+      reset_escapes!
+    end
+
+    # Auto-escape operators and regex capture vars. Appropriate for top-level syntax.
+    def reset_escapes!
+      @escape = Escapes.new(true, true)
+    end
+
+    # Auto-escape anything you'd find in a regular expression
+    def regex_mode!
+      @escape.regex = false # look for the final /
+      @escape.regex_escape = true # pass \ through to the regex engine UNLESS it's followed by a /
+      @escape.wildcards = true
+      @escape.operators = true
+      @escape.regex_capture = true
+      @escape.exp = true
+    end
+
+    def next
+      return Token.new(@i, Tokens::EOF) if @i > @end
+      char = @src[@i]
+      case char
+      when EXP_OPEN
+        @i += 1
+        token! Tokens::EXP_OPEN
+      when EXP_CLOSE
+        @i += 1
+        token! Tokens::EXP_CLOSE
+      when WILDCARD
+        @i += 1
+        token! Tokens::WILDCARD, WILDCARD
+      when REGEX_DELIM
+        if @escape.regex
+          get_str!
+        else
+          @i += 1
+          token! Tokens::REG_DELIM
+        end
+      when REGEX_CAPTURE
+        if @escape.regex_capture
+          get_str!
+        else
+          @i += 1
+          t = token! Tokens::REG_CAPTURE, @src[@i]
+          @i += 1
+          @start_i = @i
+          t
+        end
+      when OP_REPLACE, OP_APPEND, OP_PREPEND, OP_ADD, OP_SUB
+        if @escape.operators
+          get_str!
+        else
+          @i += 1
+          token! Tokens::OPERATOR, char
+        end
+      else
+        get_str!
+      end
+    end
+
+    private
+
+    def token!(type, val = nil)
+      t = Token.new(@start_i, type, val)
+      @start_i = @i
+      t
+    end
+
+    def get_str!
+      str = ""
+      escape, found_end = false, false
+      until found_end or @i > @end
+        char = @src[@i]
+
         if escape
-          tokens << Char.new(char)
+          @i += 1
+          str << char
           escape = false
           next
         end
 
         case char
-        when "\\".freeze
-          escape = true
-        when "{".freeze
-          tokens << :match_open
-        when "}".freeze
-          tokens << :match_close
-        when "*".freeze
-          tokens << :wildcard
+        when ESCAPE
+          @i += 1
+          if @escape.regex_escape and @src[@i] != REGEX_DELIM
+            str << char
+          else
+            escape = true
+          end
+        when EXP_OPEN
+          if @escape.exp
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when EXP_CLOSE
+          if @escape.exp
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when WILDCARD
+          if @escape.wildcards
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when REGEX_DELIM
+          if @escape.regex
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when REGEX_CAPTURE
+          if @escape.regex_capture
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when OP_REPLACE, OP_APPEND, OP_PREPEND, OP_ADD, OP_SUB
+          if @escape.operators
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
         else
-          tokens << Char.new(char)
+          @i += 1
+          str << char
         end
-      }
+      end
 
-      raise Error, "Trailing escape" if escape
-      tokens
+      return Token.new(@i - 1, Tokens::TR_ESC) if escape
+      token! Tokens::TEXT, str
     end
   end
 end

data/lib/fop/tokens.rb

@@ -0,0 +1,13 @@
+module Fop
+  module Tokens
+    TEXT = :TXT
+    EXP_OPEN = :"{"
+    EXP_CLOSE = :"}"
+    REG_CAPTURE = :"$"
+    REG_DELIM = :/
+    WILDCARD = :*
+    OPERATOR = :op
+    TR_ESC = :"trailing escape"
+    EOF = :EOF
+  end
+end

data/lib/fop/version.rb

@@ -1,3 +1,3 @@
 module Fop
-  VERSION = "0.1.0"
+  VERSION = "0.5.0"
 end

data/lib/fop_lang.rb

@@ -1,12 +1,22 @@
 require_relative 'fop/version'
+require_relative 'fop/compiler'
 require_relative 'fop/program'
 
 def Fop(src)
-  ::Fop::Program.new(src)
+  ::Fop.compile!(src)
 end
 
 module Fop
+  def self.compile!(src)
+    prog, errors = compile(src)
+    # TODO better exception
+    raise "Fop errors: " + errors.map(&:message).join(",") if errors
+    prog
+  end
+
   def self.compile(src)
-    Program.new(src)
+    instructions, errors = ::Fop::Compiler.compile(src)
+    return nil, errors if errors
+    return Program.new(instructions), nil
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fop_lang
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jordan Hollinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-08-20 00:00:00.000000000 Z
 dependencies: []
 description: A micro expression language for Filter and OPerations on text
 email: jordan.hollinger@gmail.com
@@ -17,10 +17,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - README.md
+- lib/fop/compiler.rb
 - lib/fop/nodes.rb
 - lib/fop/parser.rb
 - lib/fop/program.rb
 - lib/fop/tokenizer.rb
+- lib/fop/tokens.rb
 - lib/fop/version.rb
 - lib/fop_lang.rb
 homepage: https://jhollinger.github.io/fop-lang-rb/