fop_lang 0.2.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 711af4fad2907616e057049dcb84bce16ffdb381b3601a387dc7260cc16057d3
-  data.tar.gz: 40da554ca0cb21f275748593921bdc616dcf8cdfe5d4fa31494f588f2c25d66c
+  metadata.gz: 7cfe0a53d59b00d2fc60f0c0f5ed3e413eb61cf0ae445d49562f0cf861cd9f0b
+  data.tar.gz: 5aa04d8145417d52b42847be17b54a230242143486278d2826adacb7d1143101
 SHA512:
-  metadata.gz: 0253a6446b88b6de112f00a95c81f1d5f710a859998e5b3d8df21d64caecc8e013ab1edec2b68047558f8053ce135b60082078f85db1f7aa16a93b86aa487093
-  data.tar.gz: c7f0cb0387df52a3ea121e3e91a35e0444afe8862aabe054ec433d889955ee9221129fd0c07f7e36d6c71e6b9ab33f30cd16264447dfda24d29babb671112140
+  metadata.gz: 79efdf103e6e33bc508df356e2295ac49c8f112ad44a0c3aad4209bb204fadcd240b12f14f62ee3266d21762afec1e8a3262b767fda42cc7ebf019688bfbe30e
+  data.tar.gz: 1f424bf43332438a3f223af0737fa6570de8e06444669edd8808e5d41e2804fe9139598f58336ca098a1b67fba8741519375e3700916747acb8df6ddace3294d

data/README.md

@@ -1,6 +1,10 @@
 # fop_lang
 
-Fop (Filter and OPperations language) is an experimental, tiny expression language in the vein of awk and sed. This is a Ruby implementation. It is useful for simultaneously matching and transforming text input.
+Fop (Filter and OPerations language) is an experimental, tiny expression language in the vein of awk and sed. This is a Ruby implementation. It is useful for simultaneously matching and transforming text input.
+
+```ruby
+gem 'fop_lang'
+```
 
 ## Release Number Example
 
@@ -25,7 +29,7 @@ The above expression contains the only two parts of Fop (except for the wildcard
 
 **Text Literals**
 
-A text literal works how it sounds: the input must match it exactly. The only exception is the `*` (wildcard) character, which matches 0 or more of anything. Wildcards can be used anywhere except inside `{...}` (operations).
+A text literal works how it sounds: the input must match it exactly. If it matches it passes through unchanged. The only exception is the `*` (wildcard) character, which matches 0 or more of anything. Wildcards can be used anywhere except inside `{...}` (operations).
 
 If `\` (escape) is used before the special characters `*`, `{` or `}`, then that character is treated like a text literal. It's recommended to use single-quoted Ruby strings with Fop expressions that so you don't need to double-escape.
 
@@ -33,13 +37,16 @@ If `\` (escape) is used before the special characters `*`, `{` or `}`, then that
 
 Operations are the interesting part of Fop, and are specified between `{` and `}`. An Operation can consist of one to three parts:
 
-1. Matching character class (required): Defines what characters the operation will match and operate on.
+1. Matching class (required): Defines what characters the operation will match and operate on.
   * `N` is the numeric class and will match one or more digits.
   * `A` is the alpha class and will match one or more letters (lower or upper case).
   * `W` is the word class and matches alphanumeric chars and underscores.
   * `*` is the wildcard class and greedily matches everything after it.
+  * `/.../` matches on the supplied regex between the `/`'s. If you're regex contains a `/`, it must be escaped. Capture groups may be referenced in the operator argument as `$1`, `$2`, etc.
 3. Operator (optional): What to do to the matching characters.
   * `=` Replace the matching character(s) with the given argument. If no argument is given, drop the matching chars.
+  * `>` Append the following chars to the matching value.
+  * `<` Prepend the following chars to the matching value.
   * `+` Perform addition on the matching number and the argument (`N` only).
   * `-` Subtract the argument from the matching number (`N` only).
 5. Operator argument (required for some operators): meaning varies by operator.
@@ -53,6 +60,16 @@ Operations are the interesting part of Fop, and are specified between `{` and `}
   =>           'release-5.100.0'
 ```
 
+```ruby
+  f = Fop('rel{/(ease)?/}-{N=5}.{N+1}.{N=0}')
+
+  puts f.apply('release-4.99.1')
+  =>           'release-5.100.0'
+
+  puts f.apply('rel-4.99.1')
+  =>           'rel-5.100.0'
+```
+
 ```ruby
   f = Fop('release-*{N=5}.{N+100}.{N=0}')
 

data/bin/fop

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+# Used for local testing
+# $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'fop_lang'
+require 'fop/cli'
+
+opts = Fop::CLI.options!
+src = opts.src.read.chomp
+fop, errors = Fop.compile(src)
+opts.src.close
+NL = "\n".freeze
+
+if errors
+  $stderr.puts src
+  $stderr.puts errors.join(NL)
+  exit 1
+end
+
+if opts.check
+  $stdout.puts "Syntax OK" unless opts.quiet
+  exit 0
+end
+
+while (line = gets) do
+  line.chomp!
+  if (res = fop.apply(line))
+    print(res << NL)
+  end
+end

data/lib/fop/cli.rb

@@ -0,0 +1,30 @@
+require 'optparse'
+
+module Fop
+  module CLI
+    Options = Struct.new(:src, :check, :quiet)
+
+    def self.options!
+      options = Options.new
+      OptionParser.new do |opts|
+        opts.banner = "Usage: fop [options] [ 'prog' | -f progfile ] [ file ... ]"
+
+        opts.on("-fFILE", "--file=FILE", "Read program from file instead of first argument") do |f|
+          options.src = File.open(f)
+          options.src.advise(:sequential)
+        end
+
+        opts.on("-c", "--check", "Perform a syntax check on the program and exit") do
+          options.check = true
+        end
+
+        opts.on("-q", "--quiet") do
+          options.quiet = true
+        end
+      end.parse!
+
+      options.src ||= StringIO.new(ARGV.shift)
+      options
+    end
+  end
+end

data/lib/fop/compiler.rb

@@ -0,0 +1,72 @@
+require_relative 'parser'
+
+module Fop
+  module Compiler
+    def self.compile(src)
+      parser = Parser.new(src)
+      nodes, errors = parser.parse
+
+      instructions = nodes.map { |node|
+        case node
+        when Nodes::Text, Nodes::Regex
+          Instructions.regex_match(node.regex)
+        when Nodes::Expression
+          Instructions::ExpressionMatch.new(node)
+        else
+          raise "Unknown node type #{node}"
+        end
+      }
+
+      return nil, errors if errors.any?
+      return instructions, nil
+    end
+
+    module Instructions
+      BLANK = "".freeze
+      OPERATIONS = {
+        "=" => ->(_val, arg) { arg || BLANK },
+        "+" => ->(val, arg) { val.to_i + arg.to_i },
+        "-" => ->(val, arg) { val.to_i - arg.to_i },
+        ">" => ->(val, arg) { val + arg },
+        "<" => ->(val, arg) { arg + val },
+      }
+
+      def self.regex_match(regex)
+        ->(input) { input.slice! regex }
+      end
+
+      class ExpressionMatch
+        def initialize(node)
+          @regex = node.regex&.regex
+          @op = node.operator ? OPERATIONS.fetch(node.operator) : nil
+          @regex_match = node.regex_match
+          if node.arg&.any? { |a| a.is_a? Integer }
+            @arg, @arg_with_caps = nil, node.arg
+          else
+            @arg = node.arg&.join("")
+            @arg_with_caps = nil
+          end
+        end
+
+        def call(input)
+          if (match = @regex.match(input))
+            val = match.to_s
+            blank = val == BLANK
+            input.sub!(val, BLANK) unless blank
+            found_val = @regex_match || !blank
+            arg = @arg_with_caps ? sub_caps(@arg_with_caps, match.captures) : @arg
+            @op && found_val ? @op.call(val, arg) : val
+          end
+        end
+
+        private
+
+        def sub_caps(args, caps)
+          args.map { |a|
+            a.is_a?(Integer) ? caps[a].to_s : a
+          }.join("")
+        end
+      end
+    end
+  end
+end

data/lib/fop/nodes.rb

@@ -1,70 +1,30 @@
 module Fop
   module Nodes
-    Text = Struct.new(:wildcard, :str) do
-      def consume!(input)
-        @regex ||= Regexp.new((wildcard ? ".*" : "^") + Regexp.escape(str))
-        input.slice!(@regex)
-      end
-
+    Text = Struct.new(:wildcard, :str, :regex) do
       def to_s
         w = wildcard ? "*" : nil
-        "Text #{w}#{str}"
+        "[#{w}txt] #{str}"
       end
     end
 
-    Match = Struct.new(:wildcard, :tokens) do
-      NUM = "N".freeze
-      WORD = "W".freeze
-      ALPHA = "A".freeze
-      WILD = "*".freeze
-      BLANK = "".freeze
-
-      def consume!(input)
-        if (val = input.slice!(@regex))
-          @expression && val != BLANK ? @expression.call(val) : val
-        end
-      end
-
+    Regex = Struct.new(:wildcard, :src, :regex) do
       def to_s
         w = wildcard ? "*" : nil
-        @op ? "#{w}#{@match} #{@op} #{@arg}" : "#{w}#{@match}"
+        "[#{w}reg] #{src}"
       end
+    end
 
-      def parse!
-        match = tokens.shift || raise(ParserError, "Empty match")
-        raise ParserError, "Unexpected #{match}" unless match.is_a? Tokenizer::Char
-
-        @match = match.char
-        @regex =
-          case @match
-          when NUM then Regexp.new((wildcard ? ".*?" : "^") + "[0-9]+")
-          when WORD then Regexp.new((wildcard ? ".*?" : "^") + "\\w+")
-          when ALPHA then Regexp.new((wildcard ? ".*?" : "^") + "[a-zA-Z]+")
-          when WILD then /.*/
-          else raise ParserError, "Unknown match type '#{@match}'"
-          end
-
-        if (op = tokens.shift)
-          raise ParserError, "Unexpected #{op}" unless op.is_a? Tokenizer::Char
-          arg = tokens.reduce("") { |acc, t|
-            raise ParserError, "Unexpected #{t}" unless t.is_a? Tokenizer::Char
-            acc + t.char
-          }
-
-          @op = op.char
-          @arg = arg == BLANK ? nil : arg
-          @expression =
-            case @op
-            when "=" then ->(_) { @arg || BLANK }
-            when "+", "-", "*", "/"
-              raise ParserError, "Operator #{@op} is only available for numeric matches" unless @match == NUM
-              raise ParserError, "Operator #{@op} expects an argument" if @arg.nil?
-              ->(x) { x.to_i.send(@op, @arg.to_i) }
-            else raise ParserError, "Unknown operator #{@op}"
-            end
-        else
-          @op, @arg, @expression = nil, nil, nil
+    Expression = Struct.new(:wildcard, :match, :regex_match, :regex, :operator, :arg) do
+      def to_s
+        w = wildcard ? "*" : nil
+        s = "[#{w}exp] #{match}"
+        if operator
+          arg_str = arg
+            .map { |a| a.is_a?(Integer) ? "$#{a+1}" : a.to_s }
+            .join("")
+          s << " #{operator} #{arg_str}"
         end
+        s
       end
     end
   end

data/lib/fop/parser.rb

@@ -1,93 +1,162 @@
+require_relative 'tokenizer'
 require_relative 'nodes'
 
 module Fop
-  module Parser
-    Error = Class.new(StandardError)
+  class Parser
+    DIGIT = /^[0-9]$/
+    REGEX_START = "^".freeze
+    REGEX_LAZY_WILDCARD = ".*?".freeze
+    REGEX_MATCHES = {
+      "N" => "[0-9]+".freeze,
+      "W" => "\\w+".freeze,
+      "A" => "[a-zA-Z]+".freeze,
+      "*" => ".*".freeze,
+    }.freeze
+    OPS_WITH_OPTIONAL_ARGS = [Tokenizer::OP_REPLACE]
+    TR_REGEX = /.*/
 
-    def self.parse!(tokens)
-      stack = []
-      current_el = nil
+    Error = Struct.new(:type, :token, :message) do
+      def to_s
+        "#{type.to_s.capitalize} error: #{message} at column #{token.pos}"
+      end
+    end
+
+    attr_reader :errors
 
-      tokens.each { |token|
-        case current_el
-        when nil
-          current_el = new_element token
-        when :wildcard
-          current_el = new_element token, true
-          raise Error, "Unexpected * after wildcard" if current_el == :wildcard
-        when Nodes::Text
-          current_el = parse_text stack, current_el, token
-        when Nodes::Match
-          current_el = parse_match stack, current_el, token
+    def initialize(src, debug: false)
+      @tokenizer = Tokenizer.new(src)
+      @errors = []
+    end
+
+    def parse
+      nodes = []
+      wildcard = false
+      eof = false
+      # Top-level parsing. It will always be looking for a String, Regex, or Expression.
+      until eof
+        @tokenizer.reset_escapes!
+        t = @tokenizer.next
+        case t.type
+        when Tokens::WILDCARD
+          errors << Error.new(:syntax, t, "Consecutive wildcards") if wildcard
+          wildcard = true
+        when Tokens::TEXT
+          reg = build_regex!(wildcard, t, Regexp.escape(t.val))
+          nodes << Nodes::Text.new(wildcard, t.val, reg)
+          wildcard = false
+        when Tokens::EXP_OPEN
+          nodes << parse_exp!(wildcard)
+          wildcard = false
+        when Tokens::REG_DELIM
+          nodes << parse_regex!(wildcard)
+          wildcard = false
+        when Tokens::EOF
+          eof = true
         else
-          raise Error, "Unexpected token #{token} in #{current_el}"
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}")
         end
-      }
-
-      case current_el
-      when nil
-        # noop
-      when :wildcard
-        stack << Nodes::Text.new(true, "")
-      when Nodes::Text
-        stack << current_el
-      when Nodes::Match
-        raise Error, "Unclosed match"
       end
-
-      stack
+      nodes << Nodes::Text.new(true, "", TR_REGEX) if wildcard
+      return nodes, @errors
     end
 
-    private
+    def parse_exp!(wildcard = false)
+      exp = Nodes::Expression.new(wildcard)
+      parse_exp_match! exp
+      op_token = parse_exp_operator! exp
+      if exp.operator
+        parse_exp_arg! exp, op_token
+      end
+      return exp
+    end
 
-    def self.new_element(token, wildcard = false)
-      case token
-      when Tokenizer::Char
-        Nodes::Text.new(wildcard, token.char.clone)
-      when :match_open
-        Nodes::Match.new(wildcard, [])
-      when :match_close
-        raise ParserError, "Unmatched }"
-      when :wildcard
-        :wildcard
+    def parse_exp_match!(exp)
+      @tokenizer.escape.operators = false
+      t = @tokenizer.next
+      case t.type
+      when Tokens::TEXT, Tokens::WILDCARD
+        exp.match = t.val
+        if (src = REGEX_MATCHES[exp.match])
+          reg = Regexp.new((exp.wildcard ? REGEX_LAZY_WILDCARD : REGEX_START) + src)
+          exp.regex = Nodes::Regex.new(exp.wildcard, src, reg)
+        else
+          errors << Error.new(:name, t, "Unknown match type '#{exp.match}'") if exp.regex.nil?
+        end
+      when Tokens::REG_DELIM
+        exp.regex = parse_regex!(exp.wildcard)
+        exp.match = exp.regex&.src
+        exp.regex_match = true
+        @tokenizer.reset_escapes!
       else
-        raise ParserError, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string or a regex")
       end
     end
 
-    def self.parse_text(stack, text_el, token)
-      case token
-      when :match_open
-        stack << text_el
-        Nodes::Match.new(false, [])
-      when :match_close
-        raise ParserError.new, "Unexpected }"
-      when Tokenizer::Char
-        text_el.str << token.char
-        text_el
-      when :wildcard
-        stack << text_el
-        :wildcard
+    def parse_exp_operator!(exp)
+      @tokenizer.escape.operators = false
+      t = @tokenizer.next
+      case t.type
+      when Tokens::EXP_CLOSE
+        # no op
+      when Tokens::OPERATOR
+        exp.operator = t.val
       else
-        raise ParserError, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected an operator")
       end
+      t
     end
 
-    def self.parse_match(stack, match_el, token)
-      case token
-      when Tokenizer::Char
-        match_el.tokens << token
-        match_el
-      when :wildcard
-        match_el.tokens << Tokenizer::Char.new("*").freeze
-        match_el
-      when :match_close
-        match_el.parse!
-        stack << match_el
-        nil
+    def parse_exp_arg!(exp, op_token)
+      @tokenizer.escape.operators = true
+      @tokenizer.escape.regex = true
+      @tokenizer.escape.regex_capture = false if exp.regex_match
+
+      exp.arg = []
+      found_close, eof = false, false
+      until found_close or eof
+        t = @tokenizer.next
+        case t.type
+        when Tokens::TEXT
+          exp.arg << t.val
+        when Tokens::REG_CAPTURE
+          exp.arg << t.val.to_i - 1
+          errors << Error.new(:syntax, t, "Invalid regex capture; must be between 0 and 9 (found #{t.val})") unless t.val =~ DIGIT
+          errors << Error.new(:syntax, t, "Unexpected regex capture; expected str or '}'") if !exp.regex_match
+        when Tokens::EXP_CLOSE
+          found_close = true
+        when Tokens::EOF
+          eof = true
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
+        else
+          errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
+        end
+      end
+
+      if exp.arg.size != 1 and !OPS_WITH_OPTIONAL_ARGS.include?(exp.operator)
+        errors << Error.new(:arg, op_token, "Operator '#{op_token.val}' requires an argument")
+      end
+    end
+
+    def parse_regex!(wildcard)
+      @tokenizer.regex_mode!
+      t = @tokenizer.next
+      reg = Nodes::Regex.new(wildcard, t.val)
+      if t.type == Tokens::TEXT
+        reg.regex = build_regex!(wildcard, t)
       else
-        raise ParserError, "Unexpected #{token}"
+        errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string of regex")
       end
+
+      t = @tokenizer.next
+      errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected a string of regex") unless t.type == Tokens::REG_DELIM
+      reg
+    end
+
+    def build_regex!(wildcard, token, src = token.val)
+      Regexp.new((wildcard ? REGEX_LAZY_WILDCARD : REGEX_START) + src)
+    rescue RegexpError => e
+      errors << Error.new(:regex, token, e.message)
+      nil
     end
   end
 end

data/lib/fop/program.rb

@@ -1,22 +1,16 @@
-require_relative 'tokenizer'
-require_relative 'parser'
-
 module Fop
   class Program
-    attr_reader :nodes
-
-    def initialize(src)
-      tokens = Tokenizer.tokenize! src
-      @nodes = Parser.parse! tokens
+    def initialize(instructions)
+      @instructions = instructions
     end
 
     def apply(input)
       input = input.clone
       output =
-        @nodes.reduce("") { |acc, token|
-          section = token.consume!(input)
-          return nil if section.nil?
-          acc + section.to_s
+        @instructions.reduce("") { |acc, ins|
+          result = ins.call(input)
+          return nil if result.nil?
+          acc + result.to_s
         }
       input.empty? ? output : nil
     end

data/lib/fop/tokenizer.rb

@@ -1,34 +1,175 @@
+require_relative 'tokens'
+
 module Fop
-  module Tokenizer
-    Char = Struct.new(:char)
-    Error = Class.new(StandardError)
-
-    def self.tokenize!(src)
-      tokens = []
-      escape = false
-      src.each_char { |char|
+  class Tokenizer
+    Token = Struct.new(:pos, :type, :val)
+    Error = Struct.new(:pos, :message)
+    Escapes = Struct.new(:operators, :regex_capture, :regex, :regex_escape, :wildcards, :exp)
+
+    EXP_OPEN = "{".freeze
+    EXP_CLOSE = "}".freeze
+    ESCAPE = "\\".freeze
+    WILDCARD = "*".freeze
+    REGEX_DELIM = "/".freeze
+    REGEX_CAPTURE = "$".freeze
+    OP_REPLACE = "=".freeze
+    OP_APPEND = ">".freeze
+    OP_PREPEND = "<".freeze
+    OP_ADD = "+".freeze
+    OP_SUB = "-".freeze
+
+    #
+    # Controls which "mode" the tokenizer is currently in. This is a necessary result of the syntax lacking
+    # explicit string delimiters. That *could* be worked around by requiring users to escape all reserved chars,
+    # but that's ugly af. Instead, the parser continually assesses the current context and flips these flags on
+    # or off to auto-escape certain chars for the next token.
+    #
+    attr_reader :escape
+
+    def initialize(src)
+      @src = src
+      @end = src.size - 1
+      @start_i = 0
+      @i = 0
+      reset_escapes!
+    end
+
+    # Auto-escape operators and regex capture vars. Appropriate for top-level syntax.
+    def reset_escapes!
+      @escape = Escapes.new(true, true)
+    end
+
+    # Auto-escape anything you'd find in a regular expression
+    def regex_mode!
+      @escape.regex = false # look for the final /
+      @escape.regex_escape = true # pass \ through to the regex engine UNLESS it's followed by a /
+      @escape.wildcards = true
+      @escape.operators = true
+      @escape.regex_capture = true
+      @escape.exp = true
+    end
+
+    def next
+      return Token.new(@i, Tokens::EOF) if @i > @end
+      char = @src[@i]
+      case char
+      when EXP_OPEN
+        @i += 1
+        token! Tokens::EXP_OPEN
+      when EXP_CLOSE
+        @i += 1
+        token! Tokens::EXP_CLOSE
+      when WILDCARD
+        @i += 1
+        token! Tokens::WILDCARD, WILDCARD
+      when REGEX_DELIM
+        if @escape.regex
+          get_str!
+        else
+          @i += 1
+          token! Tokens::REG_DELIM
+        end
+      when REGEX_CAPTURE
+        if @escape.regex_capture
+          get_str!
+        else
+          @i += 1
+          t = token! Tokens::REG_CAPTURE, @src[@i]
+          @i += 1
+          @start_i = @i
+          t
+        end
+      when OP_REPLACE, OP_APPEND, OP_PREPEND, OP_ADD, OP_SUB
+        if @escape.operators
+          get_str!
+        else
+          @i += 1
+          token! Tokens::OPERATOR, char
+        end
+      else
+        get_str!
+      end
+    end
+
+    private
+
+    def token!(type, val = nil)
+      t = Token.new(@start_i, type, val)
+      @start_i = @i
+      t
+    end
+
+    def get_str!
+      str = ""
+      escape, found_end = false, false
+      until found_end or @i > @end
+        char = @src[@i]
+
         if escape
-          tokens << Char.new(char)
+          @i += 1
+          str << char
           escape = false
           next
         end
 
         case char
-        when "\\".freeze
-          escape = true
-        when "{".freeze
-          tokens << :match_open
-        when "}".freeze
-          tokens << :match_close
-        when "*".freeze
-          tokens << :wildcard
+        when ESCAPE
+          @i += 1
+          if @escape.regex_escape and @src[@i] != REGEX_DELIM
+            str << char
+          else
+            escape = true
+          end
+        when EXP_OPEN
+          if @escape.exp
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when EXP_CLOSE
+          if @escape.exp
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when WILDCARD
+          if @escape.wildcards
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when REGEX_DELIM
+          if @escape.regex
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when REGEX_CAPTURE
+          if @escape.regex_capture
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
+        when OP_REPLACE, OP_APPEND, OP_PREPEND, OP_ADD, OP_SUB
+          if @escape.operators
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
         else
-          tokens << Char.new(char)
+          @i += 1
+          str << char
         end
-      }
+      end
 
-      raise Error, "Trailing escape" if escape
-      tokens
+      return Token.new(@i - 1, Tokens::TR_ESC) if escape
+      token! Tokens::TEXT, str
     end
   end
 end

data/lib/fop/tokens.rb

@@ -0,0 +1,13 @@
+module Fop
+  module Tokens
+    TEXT = :TXT
+    EXP_OPEN = :"{"
+    EXP_CLOSE = :"}"
+    REG_CAPTURE = :"$"
+    REG_DELIM = :/
+    WILDCARD = :*
+    OPERATOR = :op
+    TR_ESC = :"trailing escape"
+    EOF = :EOF
+  end
+end

data/lib/fop/version.rb

@@ -1,3 +1,3 @@
 module Fop
-  VERSION = "0.2.0"
+  VERSION = "0.6.0"
 end

data/lib/fop_lang.rb

@@ -1,12 +1,22 @@
 require_relative 'fop/version'
+require_relative 'fop/compiler'
 require_relative 'fop/program'
 
 def Fop(src)
-  ::Fop::Program.new(src)
+  ::Fop.compile!(src)
 end
 
 module Fop
+  def self.compile!(src)
+    prog, errors = compile(src)
+    # TODO better exception
+    raise "Fop errors: " + errors.map(&:message).join(",") if errors
+    prog
+  end
+
   def self.compile(src)
-    Program.new(src)
+    instructions, errors = ::Fop::Compiler.compile(src)
+    return nil, errors if errors
+    return Program.new(instructions), nil
   end
 end

metadata

@@ -1,26 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: fop_lang
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Jordan Hollinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-08-20 00:00:00.000000000 Z
 dependencies: []
 description: A micro expression language for Filter and OPerations on text
 email: jordan.hollinger@gmail.com
-executables: []
+executables:
+- fop
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
+- bin/fop
+- lib/fop/cli.rb
+- lib/fop/compiler.rb
 - lib/fop/nodes.rb
 - lib/fop/parser.rb
 - lib/fop/program.rb
 - lib/fop/tokenizer.rb
+- lib/fop/tokens.rb
 - lib/fop/version.rb
 - lib/fop_lang.rb
 homepage: https://jhollinger.github.io/fop-lang-rb/