lexical_analyzer 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb89405e18a8a6774cf5e844f27d2f9006e70b79
-  data.tar.gz: b8bc6349e4ecda409ae4234d3968a49575cb0fbc
+  metadata.gz: 17cf068290c697c20216d7e8a171392caa14dc9f
+  data.tar.gz: 44cc961d916c2b03e226138ff11c0b637f54e52b
 SHA512:
-  metadata.gz: 10abdd9d749ff65755199bd82a4fc983efc870398f3c41ddc7f1f3c331361dcfc69d8c7948447932e291b5929298096612e69bc377e7354b3cc1e367db978b83
-  data.tar.gz: c9bef6e94b267ee9342267f03f1eaa1d0f03b474fd1113c88ea6169250996b7b36c2aac69437cd56bc988a859549655bd49286d6e1c211787ca9a68ccff700c0
+  metadata.gz: 80a7590b7abd987fc22cdf2be1b547a79ec2b04d4ba39d9961eb71544a03ef1604935e01b66d12da0198cb8f6e5c3e7215cb74d513b018e8ebd90307449eab82
+  data.tar.gz: 8f0483b6d822154daf2757ebeb0fb28a35ab38a0b9e42e8da10615d5d075f7f20c78b633487b10d405792e532fd3a6546ed6568344ba29bc77939c005b8fb912

data/README.md

@@ -30,44 +30,63 @@ be analyzed and an array of rules for performing that task.
 ```ruby
 lexical_analyser = LexicalAnalyzer.new(text: text, rules: rules)
 
+token = lexical_analyser.get
+
 ```
 
-#### Rules
+It is sometimes desirable to reuse an existing lexical analyzer. This can be
+done with the renew method.
+
+```ruby
+lexical_analyser.renew(text: new_text)
 
-A rule is an array with two or three elements. These elements are:
+token = lexical_analyser.get
 
-rule[0] - a symbol that represents this rule.
+```
+
+Note: The renew method takes the same arguments as the new method, text and an
+array of rules. If these are omitted, the default is to leave that value
+unchanged. The renew method returns the updated lexical analyzer just like the
+new method returns the newly created one.
 
-rule[1] - a regular expression. This must begin with a \\A clause to ensure
-correct operation of the analyzer.
+#### Rules
 
-rule[2] - an optional block that generates the output token that corresponds
-to this rule. Some examples of these blocks are:
+The rules are an array of LexicalRule objects. Each consists of a symbol, a
+regular expression, and an optional action.
 
 ```ruby
-# Ignore this input, emit no token.
-Proc.new { false }
+# Rule with default block returns [:equality, "=="] on a match.
+LexicalRule.new(:equality, /\A==/)
 
-# The default block that is used if none is given.
-lambda {|symbol, value| [symbol, value] }
+# Rule with an ignore block, ignores matches.
+LexicalRule.new(:spaces, /\A\s+/) {|_value| false }
 
-# Take the text retrieved and process it further with another analyzer.
-lambda {|_symbol, value| ka.set_text(value).get
+# Rule with an integer block returns [:integer, an_integer] on a match.
+LexicalRule.new(:integer, /\A\d+/) {|value| [@symbol, value.to_i] }
 
+# Rule with a block that expands of to a sub-rule. Returns the value of the
+# lexical analyzer in the captured variable ka.
+LexicalRule.new(:identifier, /\A[a-zA-Z_]\w*(?=\W|$|\z)/) {|value|
+  ka.renew(text: value).get
+}
 ```
 
-Note: The order of rules is important. For example, if there are two rules
+Notes:
+
+* The regular expression must begin with a \A clause to ensure correct
+operation of the analyzer.
+* The order of rules is important. For example, if there are two rules
 looking for "==" and "=" respectively, if the "=" is ahead of the "==" rule
 in the array the "==" rule will never trigger and the analysis will be
 incorrect.
 
 #### Tokens
 
-The token is also an array, with two elements.
+The output token is an array with two elements.
 
 token[0] - the symbol extracted from the rule that generated this token.
 
-token[1] - the text that generated this token.
+token[1] - the text that generated this token or its value.
 
 
 #### Example
@@ -88,7 +107,9 @@ action.
 
 #### Plan B
 
-Go to the GitHub repository and raise an issue calling attention to some
+Go to the GitHub repository and raise an
+[issue](https://github.com/PeterCamilleri/lexical_analyzer/issues)
+ calling attention to some
 aspect that could use some TLC or a suggestion or an idea.
 
 ## License

data/lib/lexical_analyzer/lexical_rule.rb

@@ -0,0 +1,24 @@
+# The Ruby Compiler Toolkit Project - Lexical Rule
+# A rule for lexical analysis.
+
+class LexicalRule
+
+  # Create a lexical rule.
+  def initialize(symbol, regex, &action)
+    @symbol = symbol
+    @regex = regex
+
+    define_singleton_method(:call, &action) if block_given?
+  end
+
+  # Does this rule match?
+  def match(text)
+    text.match(@regex)
+  end
+
+  # The default rule action.
+  def call(value)
+    [@symbol, value]
+  end
+
+end

data/lib/lexical_analyzer/version.rb

@@ -1,3 +1,3 @@
 class LexicalAnalyzer
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/lexical_analyzer.rb

@@ -1,6 +1,7 @@
 # The Ruby Compiler Toolkit Project - Lexical Analyzer
 # Scan input and extract lexical tokens.
 
+require_relative 'lexical_analyzer/lexical_rule'
 require_relative 'lexical_analyzer/version'
 
 # The RCTP class for lexical analysis.
@@ -8,26 +9,25 @@ class LexicalAnalyzer
   attr_reader   :text   # Access the text in the analyzer.
   attr_reader   :rules  # Access the array of lexical rules.
 
-  # Some array index values.
-  SYMBOL = 0
-  REGEX  = 1
-  BLOCK  = 2
-
-  # The default tokenizer block
-  DTB = lambda {|symbol, value| [symbol, value] }
-
   # Set things up.
   def initialize(text: "", rules: [])
     @text  = text
     @rules = rules
   end
 
+  # Reuse an existing lexical analyzer.
+  def renew(text: @text, rules: @rules)
+    @text  = text
+    @rules = rules
+    self
+  end
+
   # Get the next lexical token
   def get(extra=[])
     (rules + extra).each do |rule|
-      if match_data = text.match(rule[REGEX])
+      if match_data = rule.match(text)
         @text = match_data.post_match
-        return (rule[BLOCK] || DTB).call(rule[SYMBOL], match_data.to_s) || get
+        return rule.call(match_data.to_s) || get
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lexical_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - PeterCamilleri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-30 00:00:00.000000000 Z
+date: 2018-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,7 @@ files:
 - README.md
 - lexical_analyzer.gemspec
 - lib/lexical_analyzer.rb
+- lib/lexical_analyzer/lexical_rule.rb
 - lib/lexical_analyzer/version.rb
 - rakefile.rb
 - reek.txt