fop_lang 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 798fd7c335f394e878fba2f70a9f60372ea356c79f2dc63392398920d0ffce38
-  data.tar.gz: 654786ff77823e8d8dd9a348f958828346e3755e43a04a0f38e711a6c5571ea9
+  metadata.gz: e23d8d937f5a4b5e4d74010bb91923dedce019543d4d3baefc228dece938a731
+  data.tar.gz: cc97f6953b708498be169352269b861c73c9dbe52ded1a72f4370a8d18d32d48
 SHA512:
-  metadata.gz: 6761f3d7dd602d1c93a2387fc73ea14c11484e88d0d319bbf87df98925977aa15de59a63f23aafffafa384ce3b9def9f81edabae669aabc2012b00d3131e46f4
-  data.tar.gz: 7f5187cd510d691dda996284d5a400804b7573f67506701e39a6d2909c8a4026b58655f6b2800708e911377ccce790885a2238eed7a75d4873e4b599d23e67df
+  metadata.gz: e2cec9cd47a472298f7af0268a9dc03aacce374ed88da7b505e33cb4536f6f1d04107cce7c33eba4718809d54591a3111bfd26971eef3c52073ba1226be4da4f
+  data.tar.gz: 80b5700d0cdda44dd021fe48d5c134cb992c6967b10681c43488a5a7276fbf03df7d7a9427a9aa92529569eaf0d134fa789df0c8e27cd2250dc50bcb16727d13

data/README.md

@@ -1,6 +1,6 @@
 # fop_lang
 
-Fop (Filter and OPerations language) is a tiny, experimental language for filtering and transforming text. Think of it like awk but with the condition and action segments combined.
+Fop (Filter and OPerations language) is a tiny, experimental language for filtering and operating on text. Think of it like awk but with the condition and action segments combined.
 
 This is a Ruby implementation with both a library interface and a bin command.
 
@@ -16,10 +16,8 @@ You may use fop in a Ruby script:
 require 'fop_lang'
 
 f = Fop('foo {N+1}')
-
 f.apply('foo 1')
 => "foo 2"
-
 f.apply('bar 1')
 => nil
 ```
@@ -40,18 +38,24 @@ The above program demonstrates a text match, a regex match, and a match expressi
 
 ### Text match
 
+`Text ` and ` ` in the above example.
+
 The input must match this text exactly. Whitespace is part of the match. Wildcards (`*`) are allowed. Special characters (`*/{}\`) may be escaped with `\`.
 
 The output of a text match will be the matching input.
 
 ### Regex match
 
+`/(R|r)egex/` in the above example.
+
 Regular expressions may be placed between `/`s. If the regular expression contains a `/`, you may escape it with `\`. Special regex characters like `[]()+.*` may also be escaped with `\`.
 
 The output of a regex match will be the matching input.
 
 ### Match expression
 
+`{N+1}` in the above example.
+
 A match expression both matches on input and modifies that input. An expression is made up of 1 - 3 parts:
 
 1. The match, e.g. `N` for numeric.
@@ -76,6 +80,10 @@ The output of a match expression will be the _modified_ matching input. If no op
 * `+` Perform addition on the matching number and the argument (`N` only).
 * `-` Subtract the argument from the matching number (`N` only).
 
+**Whitespace**
+
+Inside of match expressions, whitespace is an optional seperator of terms, i.e. `{ N + 1 }` is the same as `{N+1}`. This means that any spaces in string arguments must be escaped. For example, replacing a word with `foo bar` looks like `{W = foo\ bar}`.
+
 ## Examples
 
 ### Release Number Example
@@ -103,10 +111,10 @@ This example takes in GitHub branch names, decides if they're release branches,
 ```
 
 ```ruby
-  f = Fop('rel{/(ease)?/}-{N=5}.{N+1}.{N=0}')
+  f = Fop('rel{/(ease)?/=}-{N=5}.{N+1}.{N=0}')
 
   puts f.apply('release-4.99.1')
-  =>           'release-5.100.0'
+  =>           'rel-5.100.0'
 
   puts f.apply('rel-4.99.1')
   =>           'rel-5.100.0'

data/lib/fop/compiler.rb

@@ -11,6 +11,8 @@ module Fop
         when Nodes::Text, Nodes::Regex
           Instructions.regex_match(node.regex)
         when Nodes::Expression
+          arg_error = Validations.validate_args(node)
+          errors << arg_error if arg_error
           Instructions::ExpressionMatch.new(node)
         else
           raise "Unknown node type #{node}"
@@ -22,13 +24,14 @@ module Fop
     end
 
     module Instructions
+      Op = Struct.new(:proc, :arity, :max_arity)
       BLANK = "".freeze
       OPERATIONS = {
-        "=" => ->(_val, arg) { arg || BLANK },
-        "+" => ->(val, arg) { val.to_i + arg.to_i },
-        "-" => ->(val, arg) { val.to_i - arg.to_i },
-        ">" => ->(val, arg) { val + arg },
-        "<" => ->(val, arg) { arg + val },
+        "=" => Op.new(->(_val, args) { args[0] || BLANK }, 0, 1),
+        "+" => Op.new(->(val, args) { val.to_i + args[0].to_i }, 1),
+        "-" => Op.new(->(val, args) { val.to_i - args[0].to_i }, 1),
+        ">" => Op.new(->(val, args) { val + args[0] }, 1),
+        "<" => Op.new(->(val, args) { args[0] + val }, 1),
       }
 
       def self.regex_match(regex)
@@ -38,14 +41,11 @@ module Fop
       class ExpressionMatch
         def initialize(node)
           @regex = node.regex&.regex
-          @op = node.operator ? OPERATIONS.fetch(node.operator) : nil
+          @op = node.operator_token ? OPERATIONS.fetch(node.operator_token.val) : nil
           @regex_match = node.regex_match
-          if node.arg&.any? { |a| a.is_a? Integer }
-            @arg, @arg_with_caps = nil, node.arg
-          else
-            @arg = node.arg&.join("")
-            @arg_with_caps = nil
-          end
+          @args = node.args&.map { |arg|
+            arg.has_captures ? arg.segments : arg.segments.join("")
+          }
         end
 
         def call(input)
@@ -54,8 +54,18 @@ module Fop
             blank = val == BLANK
             input.sub!(val, BLANK) unless blank
             found_val = @regex_match || !blank
-            arg = @arg_with_caps ? sub_caps(@arg_with_caps, match.captures) : @arg
-            @op && found_val ? @op.call(val, arg) : val
+            if @op and @args and found_val
+              args = @args.map { |arg|
+                case arg
+                when String then arg
+                when Array then sub_caps(arg, match.captures)
+                else raise "Unexpected arg type #{arg.class.name}"
+                end
+              }
+              @op.proc.call(val, args)
+            else
+              val
+            end
           end
         end
 
@@ -68,5 +78,18 @@ module Fop
         end
       end
     end
+
+    module Validations
+      def self.validate_args(exp_node)
+        op_token = exp_node.operator_token || return
+        op = Instructions::OPERATIONS.fetch(op_token.val)
+        num = exp_node.args&.size || 0
+        arity = op.arity
+        max_arity = op.max_arity || arity
+        if num < arity or num > max_arity
+          Parser::Error.new(:argument, op_token, "#{op_token.val} expects #{arity}..#{max_arity} arguments; #{num} given")
+        end
+      end
+    end
   end
 end

data/lib/fop/nodes.rb

@@ -14,18 +14,29 @@ module Fop
       end
     end
 
-    Expression = Struct.new(:wildcard, :match, :regex_match, :regex, :operator, :arg) do
+    Expression = Struct.new(:wildcard, :match, :regex_match, :regex, :operator_token, :args) do
       def to_s
         w = wildcard ? "*" : nil
         s = "[#{w}exp] #{match}"
-        if operator
-          arg_str = arg
+        if operator_token
+          arg_str = args
             .map { |a| a.is_a?(Integer) ? "$#{a+1}" : a.to_s }
             .join("")
-          s << " #{operator} #{arg_str}"
+          s << " #{operator_token.val} #{arg_str}"
         end
         s
       end
     end
+
+    Arg = Struct.new(:segments, :has_captures) do
+      def to_s
+        segments.map { |s|
+          case s
+          when Integer then "$#{s + 1}"
+          else s.to_s
+          end
+        }.join("")
+      end
+    end
   end
 end

data/lib/fop/parser.rb

@@ -12,7 +12,7 @@ module Fop
       "A" => "[a-zA-Z]+".freeze,
       "*" => ".*".freeze,
     }.freeze
-    OPS_WITH_OPTIONAL_ARGS = [Tokenizer::OP_REPLACE]
+    #OPS_WITH_OPTIONAL_ARGS = [Tokenizer::OP_REPLACE]
     TR_REGEX = /.*/
 
     Error = Struct.new(:type, :token, :message) do
@@ -63,14 +63,15 @@ module Fop
     def parse_exp!(wildcard = false)
       exp = Nodes::Expression.new(wildcard)
       parse_exp_match! exp
-      op_token = parse_exp_operator! exp
-      if exp.operator
-        parse_exp_arg! exp, op_token
+      parse_exp_operator! exp
+      if exp.operator_token
+        parse_exp_arg! exp
       end
       return exp
     end
 
     def parse_exp_match!(exp)
+      @tokenizer.escape.whitespace = false
       @tokenizer.escape.operators = false
       t = @tokenizer.next
       case t.type
@@ -93,35 +94,44 @@ module Fop
     end
 
     def parse_exp_operator!(exp)
+      @tokenizer.escape.whitespace = false
       @tokenizer.escape.operators = false
       t = @tokenizer.next
       case t.type
       when Tokens::EXP_CLOSE
         # no op
-      when Tokens::OPERATOR
-        exp.operator = t.val
+      when Tokens::OPERATOR, Tokens::TEXT
+        exp.operator_token = t
       else
         errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected an operator")
       end
-      t
     end
 
-    def parse_exp_arg!(exp, op_token)
+    def parse_exp_arg!(exp)
+      @tokenizer.escape.whitespace = false
+      @tokenizer.escape.whitespace_sep = false
       @tokenizer.escape.operators = true
       @tokenizer.escape.regex = true
       @tokenizer.escape.regex_capture = false if exp.regex_match
 
-      exp.arg = []
+      arg = Nodes::Arg.new([], false)
+      exp.args = []
       found_close, eof = false, false
       until found_close or eof
         t = @tokenizer.next
         case t.type
         when Tokens::TEXT
-          exp.arg << t.val
+          arg.segments << t.val
         when Tokens::REG_CAPTURE
-          exp.arg << t.val.to_i - 1
+          arg.has_captures = true
+          arg.segments << t.val.to_i - 1
           errors << Error.new(:syntax, t, "Invalid regex capture; must be between 0 and 9 (found #{t.val})") unless t.val =~ DIGIT
           errors << Error.new(:syntax, t, "Unexpected regex capture; expected str or '}'") if !exp.regex_match
+        when Tokens::WHITESPACE_SEP
+          if arg.segments.any?
+            exp.args << arg
+            arg = Nodes::Arg.new([])
+          end
         when Tokens::EXP_CLOSE
           found_close = true
         when Tokens::EOF
@@ -131,10 +141,11 @@ module Fop
           errors << Error.new(:syntax, t, "Unexpected #{t.type}; expected str or '}'")
         end
       end
+      exp.args << arg if arg.segments.any?
 
-      if exp.arg.size != 1 and !OPS_WITH_OPTIONAL_ARGS.include?(exp.operator)
-        errors << Error.new(:arg, op_token, "Operator '#{op_token.val}' requires an argument")
-      end
+      #if exp.arg.size != 1 and !OPS_WITH_OPTIONAL_ARGS.include?(exp.operator)
+      #  errors << Error.new(:arg, op_token, "Operator '#{op_token.val}' requires an argument")
+      #end
     end
 
     def parse_regex!(wildcard)

data/lib/fop/tokenizer.rb

@@ -3,8 +3,7 @@ require_relative 'tokens'
 module Fop
   class Tokenizer
     Token = Struct.new(:pos, :type, :val)
-    Error = Struct.new(:pos, :message)
-    Escapes = Struct.new(:operators, :regex_capture, :regex, :regex_escape, :wildcards, :exp)
+    Escapes = Struct.new(:whitespace, :whitespace_sep, :operators, :regex_capture, :regex, :regex_escape, :wildcards, :exp)
 
     EXP_OPEN = "{".freeze
     EXP_CLOSE = "}".freeze
@@ -17,6 +16,7 @@ module Fop
     OP_PREPEND = "<".freeze
     OP_ADD = "+".freeze
     OP_SUB = "-".freeze
+    WHITESPACE = " ".freeze
 
     #
     # Controls which "mode" the tokenizer is currently in. This is a necessary result of the syntax lacking
@@ -36,11 +36,12 @@ module Fop
 
     # Auto-escape operators and regex capture vars. Appropriate for top-level syntax.
     def reset_escapes!
-      @escape = Escapes.new(true, true)
+      @escape = Escapes.new(true, true, true, true)
     end
 
     # Auto-escape anything you'd find in a regular expression
     def regex_mode!
+      @escape.whitespace = true
       @escape.regex = false # look for the final /
       @escape.regex_escape = true # pass \ through to the regex engine UNLESS it's followed by a /
       @escape.wildcards = true
@@ -86,6 +87,17 @@ module Fop
           @i += 1
           token! Tokens::OPERATOR, char
         end
+      when WHITESPACE
+        if @escape.whitespace
+          get_str!
+        elsif !@escape.whitespace_sep
+          @i += 1
+          token! Tokens::WHITESPACE_SEP
+        else
+          @i += 1
+          @start_i = @i
+          self.next
+        end
       else
         get_str!
       end
@@ -162,6 +174,13 @@ module Fop
           else
             found_end = true
           end
+        when WHITESPACE
+          if @escape.whitespace
+            @i += 1
+            str << char
+          else
+            found_end = true
+          end
         else
           @i += 1
           str << char

data/lib/fop/tokens.rb

@@ -8,6 +8,7 @@ module Fop
     WILDCARD = :*
     OPERATOR = :op
     TR_ESC = :"trailing escape"
+    WHITESPACE_SEP = :s
     EOF = :EOF
   end
 end

data/lib/fop/version.rb

@@ -1,3 +1,3 @@
 module Fop
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fop_lang
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jordan Hollinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2021-09-01 00:00:00.000000000 Z
 dependencies: []
 description: A micro expression language for Filter and OPerations on text
 email: jordan.hollinger@gmail.com