lexical_analyzer 0.3.0 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 17cf068290c697c20216d7e8a171392caa14dc9f
-  data.tar.gz: 44cc961d916c2b03e226138ff11c0b637f54e52b
+SHA256:
+  metadata.gz: 183cf3a641ff2b13b77349e4f57ba380f48d4ff41590186b5759ed6e4cea1a0a
+  data.tar.gz: cc38ee450153021a29fe4dd8229f5f62ad1e30b2116359d0117a98f05173e77e
 SHA512:
-  metadata.gz: 80a7590b7abd987fc22cdf2be1b547a79ec2b04d4ba39d9961eb71544a03ef1604935e01b66d12da0198cb8f6e5c3e7215cb74d513b018e8ebd90307449eab82
-  data.tar.gz: 8f0483b6d822154daf2757ebeb0fb28a35ab38a0b9e42e8da10615d5d075f7f20c78b633487b10d405792e532fd3a6546ed6568344ba29bc77939c005b8fb912
+  metadata.gz: bfeb055736ade9ddcc4ec429f072fb51e8f42684beb22b6a4af21230f89bc6b1d70c7be6750d3c76c9d6ac02b20ad6192376712874765700e7de831ea7ac5082
+  data.tar.gz: d82439f9985d528010ebf4c3dad7aec350ed6b660e6bb840a4fa733861f0cf906d05204a6fcb3310176ca24836eb9348e85eee7d06923b43bc68f7176ab21499

data/README.md

@@ -1,10 +1,21 @@
 # LexicalAnalyzer
 
-The lexical analyzer is a component of the Ruby Compiler Toolkit Project that
-scans an input text against an array of rules and generating the lexical
-tokens that it detects. It is normally used in conjunction with a parse queue
-object which handles queuing of tokens and back tracking of the compile process
-when needed.
+The lexical analyzer is a component of the Ruby Compiler Toolkit Project (rctp)
+that scans an input text against an array of rules and generating the lexical
+tokens that it detects. This process is shown below:
+
+![The Lexical Process](./images/lexical_process.png)
+
+In general, each time the lexical_analyzer receives the "get" message, it tries
+to extract another token from the source text. As such, the lexical analyzer
+gem component is the first stage of the compilation process for a compiler
+built using the rctp. With its array of lexical rules it provides the language
+tokens needed to operate the compiler's parser.
+
+The lexical analyzer is normally used in conjunction with a parse queue object
+which handles queuing of tokens and back tracking of the compile process when
+needed. In the rcpt this is done by the gem
+[parse_queue](https://github.com/PeterCamilleri/parse_queue).
 
 ## Installation
 
@@ -58,27 +69,32 @@ regular expression, and an optional action.
 # Rule with default block returns [:equality, "=="] on a match.
 LexicalRule.new(:equality, /\A==/)
 
+# Rule with block equivalent to the default.
+LexicalRule.new(:lparen, /\A\(/) {|value| [symbol, value]}
+
 # Rule with an ignore block, ignores matches.
 LexicalRule.new(:spaces, /\A\s+/) {|_value| false }
 
 # Rule with an integer block returns [:integer, an_integer] on a match.
-LexicalRule.new(:integer, /\A\d+/) {|value| [@symbol, value.to_i] }
+LexicalRule.new(:integer, /\A\d+/) {|value| [symbol, value.to_i] }
 
 # Rule with a block that expands of to a sub-rule. Returns the value of the
 # lexical analyzer in the captured variable ka.
-LexicalRule.new(:identifier, /\A[a-zA-Z_]\w*(?=\W|$|\z)/) {|value|
-  ka.renew(text: value).get
-}
+LexicalRule.new(:identifier, /\A[a-zA-Z_]\w*(?=\W|$|\z)/) {|value| ka.renew(text: value).get}
 ```
 
 Notes:
 
 * The regular expression must begin with a \A clause to ensure correct
 operation of the analyzer.
+* An exception to the above is the use of the expression /.+/ at the end of the
+rule list as a sort of lexical "else" catch-all clause.
 * The order of rules is important. For example, if there are two rules
 looking for "==" and "=" respectively, if the "=" is ahead of the "==" rule
 in the array the "==" rule will never trigger and the analysis will be
 incorrect.
+* The method LexicalRule#symbol is a read accessor for the symbol property of
+the lexical rule.
 
 #### Tokens
 

data/lexical_analyzer.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>=2.3.0'
 
-  spec.add_development_dependency "bundler", "~> 1.15"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 2.1.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency 'reek', "~> 5.0.2"
 end

data/lib/lexical_analyzer.rb

@@ -6,8 +6,8 @@ require_relative 'lexical_analyzer/version'
 
 # The RCTP class for lexical analysis.
 class LexicalAnalyzer
-  attr_reader   :text   # Access the text in the analyzer.
-  attr_reader   :rules  # Access the array of lexical rules.
+  attr_reader :text   # Access the text in the analyzer.
+  attr_reader :rules  # Access the array of lexical rules.
 
   # Set things up.
   def initialize(text: "", rules: [])
@@ -23,8 +23,8 @@ class LexicalAnalyzer
   end
 
   # Get the next lexical token
-  def get(extra=[])
-    (rules + extra).each do |rule|
+  def get(extra=nil)
+    (extra ? rules + extra : rules).each do |rule|
       if match_data = rule.match(text)
         @text = match_data.post_match
         return rule.call(match_data.to_s) || get

data/lib/lexical_analyzer/lexical_rule.rb

@@ -1,24 +1,26 @@
-# The Ruby Compiler Toolkit Project - Lexical Rule
-# A rule for lexical analysis.
-
-class LexicalRule
-
-  # Create a lexical rule.
-  def initialize(symbol, regex, &action)
-    @symbol = symbol
-    @regex = regex
-
-    define_singleton_method(:call, &action) if block_given?
-  end
-
-  # Does this rule match?
-  def match(text)
-    text.match(@regex)
-  end
-
-  # The default rule action.
-  def call(value)
-    [@symbol, value]
-  end
-
-end
+# The Ruby Compiler Toolkit Project - Lexical Rule
+# A rule for lexical analysis.
+
+# The RCTP class for lexical rules.
+class LexicalRule
+  attr_reader :symbol  # Read access to the rule's symbol.
+  attr_reader :regex   # Read access to the rule's regex.
+
+  # Create a lexical rule.
+  def initialize(symbol, regex, &action)
+    @symbol = symbol
+    @regex  = regex
+
+    define_singleton_method(:call, &action) if block_given?
+  end
+
+  # Does this rule match?
+  def match(text)
+    text.match(@regex)
+  end
+
+  # The default rule action.
+  def call(value)
+    [symbol, value]
+  end
+end

data/lib/lexical_analyzer/version.rb

@@ -1,3 +1,3 @@
 class LexicalAnalyzer
-  VERSION = "0.3.0"
+  VERSION = "0.3.5".freeze
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: lexical_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - PeterCamilleri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-03 00:00:00.000000000 Z
+date: 2021-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: 2.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +78,7 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
+- images/lexical_process.png
 - lexical_analyzer.gemspec
 - lib/lexical_analyzer.rb
 - lib/lexical_analyzer/lexical_rule.rb
@@ -103,8 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: The RCTP lexical analyser.