skeem 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31d8e575d516bf4429e18814019ca54198bd1088
-  data.tar.gz: 35a3d39adf7748b70c07cc1553a1b35ec94cd108
+  metadata.gz: a4521e994b28e81e3fa1f8852bf096c2e375def5
+  data.tar.gz: 86e7e0738987b88b9043740632b510e5bee6a49f
 SHA512:
-  metadata.gz: a440baab7b58725760e00511c68c879fed77a6fc168fbafd89f361fa3fc3b49ba69cc5901a77438e4eacbcb2942679b6247d67ae02dec8e44d4145f003ed8eff
-  data.tar.gz: 21457dcdd46a2cf2efc1ac4d184be50748429954f79de03480f1c6dd26094a08fda17c321cf231a7954bcac3c95593ff75a89238e0e1e97b23d69058a92ab68c
+  metadata.gz: 0197b9df68cba199a243b53d22f329728575ec2c8021b813f6f3eb4be4e39d530cfc53fa221be7994a636da06fdc1ad17977985d7d4e0c54dd3f4dda1c8b6cae
+  data.tar.gz: 493fb5dd6786dda700abf79ea211d96b832772540986b36eca887ddb2abba4816954080fb5508fdc55dc9ab4b375b5b129f031f507efcf9caf081e05b28ba10f

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## [0.0.2] - 2018-08-25
+### Changed
+- Class`Tokenizer` improved, does recognize delimiters, booleans, integers, real numbers, strings, and identifiers.
+- Spec file `Tokenizer_spec.rb` expanded with more tests.
+
 ## [0.0.1] - 2018-08-25
 ### Added
 - Initial `Tokenizer` class commit

data/README.md

@@ -31,6 +31,9 @@ Roadmap:
 - Make it pass all examples from the [Reasoned Schemer](https://mitpress.mit.edu/books/reasoned-schemer-second-edition) book.
 
 TODO: Write usage instructions here
+
+Good to know:
+Online book: [The Scheme Programming Language (4th Ed.)](https://www.scheme.com/tspl4/)
   
 ## Development
 

data/lib/skeem/tokenizer.rb

@@ -30,7 +30,14 @@ module Skeem
     # Constructor. Initialize a tokenizer for Skeem.
     # @param source [String] Skeem text to tokenize.
     def initialize(source)
-      @scanner = StringScanner.new(source)
+      @scanner = StringScanner.new('')
+      reinitialize(source)
+    end
+
+
+    # @param source [String] Skeem text to tokenize.
+    def reinitialize(source)
+      @scanner.string = source
       @lineno = 1
       @line_start = 0
     end
@@ -58,19 +65,30 @@ module Skeem
       if "()'`".include? curr_ch
         # Delimiters, separators => single character token
         token = build_token(@@lexeme2name[curr_ch], scanner.getch)
-      elsif (lexeme = scanner.scan(/#(?:t|f|true|false)((?=\s|[|()";])|$)/))
+      elsif (lexeme = scanner.scan(/#(?:\.)(?=\s|[|()";]|$)/)) # Single char occurring alone
+        token = build_token('PERIOD', lexeme)
+      elsif (lexeme = scanner.scan(/#(?:t|f|true|false)(?=\s|[|()";]|$)/))
         token = build_token('BOOLEAN', lexeme) # normalized lexeme
-      elsif (lexeme = scanner.scan(/[0-9]+((?=\s|[|()";])|$)/))
+      elsif (lexeme = scanner.scan(/[+-]?[0-9]+(?=\s|[|()";]|$)/))
         token = build_token('INTEGER', lexeme) # Decimal radix
-      elsif (lexeme = scanner.scan(/-?[0-9]+(\.[0-9]+)?((?=\s|[|()";])|$)/))
+      elsif (lexeme = scanner.scan(/[+-]?[0-9]+\.[0-9]+(?:(?:e|E)[+-]?[0-9]+)?/))
         token = build_token('REAL', lexeme)
       elsif (lexeme = scanner.scan(/"(?:\\"|[^"])*"/)) # Double quotes literal?
         unquoted = lexeme.gsub(/(^")|("$)/, '')
         token = build_token('STRING_LIT', unquoted)
-      elsif (lexeme = scanner.scan(/([\+\-])((?=\s|[|()";])|$)/))
-        token = build_token('IDENTIFIER', lexeme) # Plus and minus as identifiers
       elsif (lexeme = scanner.scan(/[a-zA-Z!$%&*\/:<=>?@^_~][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
         token = build_token('IDENTIFIER', lexeme)
+      elsif (lexeme = scanner.scan(/\|(?:[^|])*\|/)) # Vertical bar delimited
+        token = build_token('IDENTIFIER', lexeme)
+      elsif (lexeme = scanner.scan(/([\+\-])((?=\s|[|()";])|$)/))
+        #  # R7RS peculiar identifiers case 1: isolated plus and minus as identifiers
+        token = build_token('IDENTIFIER', lexeme)
+      elsif (lexeme = scanner.scan(/[+-][a-zA-Z!$%&*\/:<=>?@^_~+-@][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
+        # R7RS peculiar identifiers case 2
+        token = build_token('IDENTIFIER', lexeme)
+      elsif (lexeme = scanner.scan(/\.[a-zA-Z!$%&*\/:<=>?@^_~+-@.][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
+        # R7RS peculiar identifiers case 4
+        token = build_token('IDENTIFIER', lexeme)
       else # Unknown token
         erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
         sequel = scanner.scan(/.{1,20}/)

data/lib/skeem/version.rb

@@ -1,3 +1,3 @@
 module Skeem
-  VERSION = '0.0.1'.freeze
+  VERSION = '0.0.2'.freeze
 end

data/spec/skeem/tokenizer_spec.rb

@@ -10,7 +10,12 @@ module Skeem
         expect(token.lexeme).to eq(lexeme)
       end
     end
+    
+    def unquoted(aString)
+      aString.gsub(/(^")|("$)/, '')
+    end
 
+    # Default instantiation
     subject { Tokenizer.new('') }
 
     context 'Initialization:' do
@@ -21,10 +26,11 @@ module Skeem
       it 'should have its scanner initialized' do
         expect(subject.scanner).to be_kind_of(StringScanner)
       end
-      
+    end # context
+
     context 'Delimiter and separator token recognition:' do
       it 'should tokenize single char delimiters' do
-        subject.scanner.string = "( ) ' `"
+        subject.reinitialize("( ) ' `")
         tokens = subject.tokens
         tokens.each { |token| expect(token).to be_kind_of(SToken) }
         terminals = tokens.map(&:terminal)
@@ -32,6 +38,122 @@ module Skeem
         expect(terminals).to eq(prediction)
       end
     end # context
+
+    context 'Boolean literals recognition:' do
+      it 'should tokenize boolean constants' do
+        tests = [
+          # couple [raw input, expected]
+          ['#t', '#t'],
+          [' #f', '#f'],
+          ['#true ', '#true'],
+          [' #false', '#false']
+        ]
+
+        tests.each do |(input, prediction)|
+          subject.reinitialize(input)
+          token = subject.tokens.first
+          expect(token.terminal).to eq('BOOLEAN')
+          expect(token.lexeme).to eq(prediction)
+        end
+      end
+    end # context
+
+    context 'Integer literals recognition:' do
+      it 'should tokenize integers in default radix 10' do
+        tests = [
+          # couple [raw input, expected]
+          ['0', '0'],
+          [' 3', '3'],
+          ['+3 ', '+3'],
+          ['-3', '-3'],
+          ['-1234', '-1234']
+        ]
+
+        tests.each do |(input, prediction)|
+          subject.reinitialize(input)
+          token = subject.tokens.first
+          expect(token.terminal).to eq('INTEGER')
+          expect(token.lexeme).to eq(prediction)
+        end
+      end
+    end # context
+
+    context 'Real number recognition:' do
+      it 'should tokenize real numbers' do
+        tests = [
+          # couple [raw input, expected]
+          ["\t\t3.45e+6", '3.45e+6'],
+          ['+3.45e+6', '+3.45e+6'],
+          ['-3.45e+6', '-3.45e+6']
+        ]
+
+        tests.each do |(input, prediction)|
+          subject.reinitialize(input)
+          token = subject.tokens.first
+          expect(token.terminal).to eq('REAL')
+          expect(token.lexeme).to eq(prediction)
+        end
+      end
+    end # context
+
+    context 'String recognition:' do
+      it 'should tokenize strings' do
+        examples = [
+          # Some examples taken from R7RS document
+          '"Hello world!"',
+          '"The word \"recursion\" has many meanings."'
+        ]
+
+        examples.each do |input|
+          # puts input
+          subject.reinitialize(input)
+          token = subject.tokens.first
+          expect(token.terminal).to eq('STRING_LIT')
+          expect(token.lexeme).to eq(unquoted(input))
+        end
+      end
+    end # context
+
+=begin
+For later:
+"Another example:\ntwo lines of text"
+"Here's text \
+containing just one line"
+"\x03B1; is named GREEK SMALL LETTER ALPHA."
+=end
+
+    context 'Identifier recognition:' do
+      it 'should tokenize identifier' do
+        examples = [
+          # Examples taken from R7RS document
+          '...', '+', '+soup+', '<=?',
+          '->string', 'a34kTMNs', 'lambda',
+          'list->vector', 'q', 'V17a',
+          '|two words|', '|two\x20;words|',
+          'the-word-recursion-has-many-meanings'
+        ]
+
+        examples.each do |input|
+          subject.reinitialize(input)
+          token = subject.tokens.first
+          expect(token.terminal).to eq('IDENTIFIER')
+          expect(token.lexeme).to eq(input)
+        end
+      end
+    end # context
+    
+    context 'Scanning Scheme sample code' do
+      it 'should read examples from lis.py page' do
+        source = <<-SCHEME
+(if (> (val x) 0) 
+    (fn (+ (aref A i) (* 3 i)) 
+        (quote (one two)))
+      end
+    end
+SCHEME
+        subject.reinitialize(source)
+        expect { subject.tokens }.not_to raise_error
+      end
     end # context
   end # describe
 end # module

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skeem
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Dimitri Geshef