lexical_analyzer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b885cc4571ef000004744fa8763ed499cb484fb
-  data.tar.gz: 1e6433c47d02a82e7e0f7c7e0f989180e6f800c3
+  metadata.gz: 54b488be7553b7be24ea6fa73c51fb35f13ad33d
+  data.tar.gz: 42df79a772749f16ab5af6b902b84897086385dc
 SHA512:
-  metadata.gz: 1dfe50010426de1324068672586199d9ed9b6aa7658a982a261635731813f305ee0772aeb4a8e15658b89721873868b1a6a32a354143c3dc53d6fb4306f6fb1d
-  data.tar.gz: abc638957087cf9408f36dc5199287bf32e8187cfc46ec320a7525fbf5ac1a38f6ed9aba033bf983c339adb9dc298cf5df7357a7c16be27081137ab18242b9f4
+  metadata.gz: 41c89261b16eb1b5f626e5d824a59d1a84e0aad0d11eda7e15454fcc243f3f6c1e85645d96e7caa4b06fce90ff0007ad1796d84d15391e160df4e3f06f9a1c82
+  data.tar.gz: e9352a9008ae1353193c277162e0b9979e89e495c2dfa5d78e9407d088b612d483f76603dea29e0d0c4650f72dcca0aa95704613ed60993b311b744d825acda1

data/README.md

@@ -1,6 +1,10 @@
 # LexicalAnalyzer
 
-WIP
+The lexical analyzer is a component of the Ruby Compiler Toolkit Project that
+scans an input text against an array of rules and generating the lexical
+tokens that it detects. It is normally used in conjunction with a parse queue
+object which handles queuing of tokens and back tracking of the compile process
+when needed.
 
 ## Installation
 
@@ -20,11 +24,72 @@ Or install it yourself as:
 
 ## Usage
 
-WIP
+A lexical analyzer object is created with two keyword parameters, the text to
+be analyzed and an array of rules for performing that task.
+
+```ruby
+lexical_analyser = LexicalAnalyzer.new(text: text, rules: rules)
+
+```
+
+#### Rules
+
+A rule is an array with two or three elements. These elements are:
+
+rule[0] - a symbol that represents this rule.
+
+rule[1] - a regular expression. This must begin with a \\A clause to ensure
+correct operation of the analyzer.
+
+rule[2] - an optional block that generates the output token that corresponds
+to this rule. Some examples of these blocks are:
+
+```ruby
+# Ignore this input, emit no token.
+Proc.new { false }
+
+# The default block that is used if none is given.
+lambda {|symbol, value| [symbol, value] }
+
+# Take the text retrieved and process it further with another analyzer.
+lambda {|_symbol, value| ka.set_text(value).get
+
+```
+
+Note: The order of rules is important. For example, if there are two rules
+looking for "==" and "=" respectively, if the "=" is ahead of the "==" rule
+in the array the "==" rule will never trigger and the analysis will be
+incorrect.
+
+#### Tokens
+
+The token is also an array, with two elements.
+
+token[0] - the symbol extracted from the rule that generated this token.
+
+token[1] - the text that generated this token.
+
+
+#### Example
+
+The test file "lexical_analyzer_test.rb" has the method
+test_some_lexical_analyzing that is a really good example of this gem in
+action.
 
 ## Contributing
 
-WIP
+#### Plan A
+
+1. Fork it ( https://github.com/PeterCamilleri/lexical_analyzer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+#### Plan B
+
+Go to the GitHub repository and raise an issue calling attention to some
+aspect that could use some TLC or a suggestion or an idea.
 
 ## License
 

data/lib/lexical_analyzer.rb

@@ -1,44 +1,35 @@
-# coding: utf-8
-
-require_relative 'lexical_analyzer/version'
-
 # The Ruby Compiler Toolkit Project - Lexical Analyzer
 # Scan input and extract lexical tokens.
 
-class LexicalAnalyzer
+require_relative 'lexical_analyzer/version'
 
-  # Access the text in the analyzer.
-  attr_reader :text
+class LexicalAnalyzer
+  attr_accessor :text   # Access the text in the analyzer.
+  attr_reader   :rules  # Access the array of lexical rules.
 
-  # Access the array of lexical rules.
-  attr_reader :rules
+  # Some array index values.
+  SYMBOL = 0
+  REGEX  = 1
+  BLOCK  = 2
 
   # The default tokenizer block
   DTB = lambda {|symbol, value| [symbol, value] }
 
   # Set things up.
   def initialize(text: "", rules: [])
-    @text = text
+    @text  = text
     @rules = rules
   end
 
-  # Set the text.
-  def set_text(text)
-    @text = text
-    self
-  end
-
   # Get the next lexical token
   def get
     rules.each do |rule|
-      if match_data = text.match(rule[1])
+      if match_data = text.match(rule[REGEX])
         @text = match_data.post_match
-
-        return (rule[2] || DTB).call(rule[0], match_data.to_s) || get
+        return (rule[BLOCK] || DTB).call(rule[SYMBOL], match_data.to_s) || get
       end
     end
 
     false
   end
-
 end

data/lib/lexical_analyzer/version.rb

@@ -1,5 +1,3 @@
-# coding: utf-8
-
 class LexicalAnalyzer
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/rakefile.rb

@@ -8,3 +8,9 @@ Rake::TestTask.new(:test) do |t|
 end
 
 task :default => :test
+
+desc "What version of lexical analyzer is this?"
+task :vers do |t|
+  puts
+  puts "lexical_analyzer version = #{LexicalAnalyzer::VERSION}"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lexical_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - PeterCamilleri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-25 00:00:00.000000000 Z
+date: 2018-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler