skeem 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39b492f6106bf72c46fdcd5fd148bd91a4797835
-  data.tar.gz: 5b2be1ee0f1ab32df436f71dda5e6bf704420abf
+  metadata.gz: 31d8e575d516bf4429e18814019ca54198bd1088
+  data.tar.gz: 35a3d39adf7748b70c07cc1553a1b35ec94cd108
 SHA512:
-  metadata.gz: 3aa7a80bc07408bf2c659d5ef5c02669efcaa78bac354831c7d92c41a95d029f2d5ada637e7635937705d9569b684a3380db90667882b696caa5fd3659fb8cd7
-  data.tar.gz: 8c1a58c9d7991020d6456bdf1dc776fa19a5251e904a5006f55028179247ffe2560cda207fae8e79155daaf2ede741a78a92bd73d4b50849a63ee260c724c0f3
+  metadata.gz: a440baab7b58725760e00511c68c879fed77a6fc168fbafd89f361fa3fc3b49ba69cc5901a77438e4eacbcb2942679b6247d67ae02dec8e44d4145f003ed8eff
+  data.tar.gz: 21457dcdd46a2cf2efc1ac4d184be50748429954f79de03480f1c6dd26094a08fda17c321cf231a7954bcac3c95593ff75a89238e0e1e97b23d69058a92ab68c

data/.rspec

@@ -1,3 +1,3 @@
---format documentation
+--backtrace
 --color
 --require spec_helper

data/CHANGELOG.md

@@ -1,6 +1,10 @@
+## [0.0.1] - 2018-08-25
+### Added
+- Initial `Tokenizer` class commit
+
 ## [0.0.0] - 2018-08-24
 ### Added
-- Initial Github commit
+- Initial Github commit 
 
 ## Unreleased
 ### Added

data/lib/skeem/stoken.rb

@@ -0,0 +1,22 @@
+require 'rley' # Load the Rley gem
+
+module Skeem
+  Position = Struct.new(:line, :column) do
+    def to_s
+      "line #{line}, column #{column}"
+    end
+  end
+
+  # Specialization of Token class.
+  # It stores the position in (line, row) of the token
+  class SToken < Rley::Lexical::Token
+    attr_reader(:position)
+
+    def initialize(theLexeme, aTerminal, aPosition)
+      super(theLexeme, aTerminal)
+      @position = aPosition
+    end
+  end # class
+end # module
+
+# End of file

data/lib/skeem/tokenizer.rb

@@ -0,0 +1,118 @@
+# File: tokenizer.rb
+# Tokenizer for Skeem language (a small subset of Scheme)
+require 'strscan'
+require_relative 'stoken'
+
+module Skeem
+  # A tokenizer for the Skeem dialect.
+  # Responsibility: break Skeem input into a sequence of token objects.
+  # The tokenizer should recognize:
+  # Identifiers:
+  # Integer literals including single digit
+  # String literals (quote delimited)
+  # Single character literal
+  # Delimiters: parentheses '(',  ')'
+  # Separators: comma
+  class Tokenizer
+    attr_reader(:scanner)
+    attr_reader(:lineno)
+    attr_reader(:line_start)
+
+    @@lexeme2name = {
+      "'" => 'APOSTROPHE',
+      '`' => 'BACKQUOTE',
+      '(' => 'LPAREN',
+      ')' => 'RPAREN'
+    }.freeze
+
+    class ScanError < StandardError; end
+
+    # Constructor. Initialize a tokenizer for Skeem.
+    # @param source [String] Skeem text to tokenize.
+    def initialize(source)
+      @scanner = StringScanner.new(source)
+      @lineno = 1
+      @line_start = 0
+    end
+
+    # @return [Array<SToken>] | Returns a sequence of tokens
+    def tokens
+      tok_sequence = []
+      until @scanner.eos?
+        token = _next_token
+        tok_sequence << token unless token.nil?
+      end
+
+      return tok_sequence
+    end
+
+    private
+
+    def _next_token
+      skip_whitespaces
+      curr_ch = scanner.peek(1)
+      return nil if curr_ch.nil? || curr_ch.empty?
+
+      token = nil
+
+      if "()'`".include? curr_ch
+        # Delimiters, separators => single character token
+        token = build_token(@@lexeme2name[curr_ch], scanner.getch)
+      elsif (lexeme = scanner.scan(/#(?:t|f|true|false)((?=\s|[|()";])|$)/))
+        token = build_token('BOOLEAN', lexeme) # normalized lexeme
+      elsif (lexeme = scanner.scan(/[0-9]+((?=\s|[|()";])|$)/))
+        token = build_token('INTEGER', lexeme) # Decimal radix
+      elsif (lexeme = scanner.scan(/-?[0-9]+(\.[0-9]+)?((?=\s|[|()";])|$)/))
+        token = build_token('REAL', lexeme)
+      elsif (lexeme = scanner.scan(/"(?:\\"|[^"])*"/)) # Double quotes literal?
+        unquoted = lexeme.gsub(/(^")|("$)/, '')
+        token = build_token('STRING_LIT', unquoted)
+      elsif (lexeme = scanner.scan(/([\+\-])((?=\s|[|()";])|$)/))
+        token = build_token('IDENTIFIER', lexeme) # Plus and minus as identifiers
+      elsif (lexeme = scanner.scan(/[a-zA-Z!$%&*\/:<=>?@^_~][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
+        token = build_token('IDENTIFIER', lexeme)
+      else # Unknown token
+        erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+        sequel = scanner.scan(/.{1,20}/)
+        erroneous += sequel unless sequel.nil?
+        raise ScanError, "Unknown token #{erroneous} on line #{lineno}"
+      end
+
+      return token
+    end
+
+    def build_token(aSymbolName, aLexeme)
+      begin
+        col = scanner.pos - aLexeme.size - @line_start + 1
+        pos = Position.new(@lineno, col)
+        token = SToken.new(aLexeme, aSymbolName, pos)
+      rescue StandardError => exc
+        puts "Failing with '#{aSymbolName}' and '#{aLexeme}'"
+        raise exc
+      end
+
+      return token
+    end
+
+    def skip_whitespaces
+      pre_pos = scanner.pos
+
+      loop do
+        ws_found = false
+        found = scanner.skip(/[ \t\f]+/)
+        ws_found = true if found
+        found = scanner.skip(/(?:\r\n)|\r|\n/)
+        if found
+          ws_found = true
+          @lineno += 1
+          @line_start = scanner.pos
+        end
+        break unless ws_found
+      end
+
+      curr_pos = scanner.pos
+      return if curr_pos == pre_pos
+    end
+  end # class
+end # module
+# End of file

data/lib/skeem/version.rb

@@ -1,3 +1,3 @@
 module Skeem
-  VERSION = '0.0.0'.freeze
+  VERSION = '0.0.1'.freeze
 end

data/skeem.gemspec

@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Dimitri Geshef']
   spec.email         = ['famished.tiger@yahoo.com']
 
-  spec.description       = <<-DESCR
+  spec.description = <<-DESCR
   Skeem is an interpreter of a subset of the Scheme programming language.
 DESCR
   spec.summary = <<-SUMMARY
@@ -53,6 +53,10 @@ SUMMARY
   spec.require_paths = ['lib']
   PkgExtending.pkg_files(spec)
   PkgExtending.pkg_documentation(spec)
+  # Runtime dependencies
+  spec.add_dependency 'rley', '~> 0.6'
+
+  # Development dependencies
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'

data/spec/skeem/tokenizer_spec.rb

@@ -0,0 +1,37 @@
+require_relative '../spec_helper' # Use the RSpec framework
+require_relative '../../lib/skeem/tokenizer' # Load the class under test
+
+module Skeem
+  describe Tokenizer do
+    def match_expectations(aTokenizer, theExpectations)
+      aTokenizer.tokens.each_with_index do |token, i|
+        terminal, lexeme = theExpectations[i]
+        expect(token.terminal).to eq(terminal)
+        expect(token.lexeme).to eq(lexeme)
+      end
+    end
+
+    subject { Tokenizer.new('') }
+
+    context 'Initialization:' do
+      it 'should be initialized with a text to tokenize' do
+        expect { Tokenizer.new('(+ 2 3)') }.not_to raise_error
+      end
+
+      it 'should have its scanner initialized' do
+        expect(subject.scanner).to be_kind_of(StringScanner)
+      end
+      
+    context 'Delimiter and separator token recognition:' do
+      it 'should tokenize single char delimiters' do
+        subject.scanner.string = "( ) ' `"
+        tokens = subject.tokens
+        tokens.each { |token| expect(token).to be_kind_of(SToken) }
+        terminals = tokens.map(&:terminal)
+        prediction = %w[LPAREN RPAREN APOSTROPHE BACKQUOTE]
+        expect(terminals).to eq(prediction)
+      end
+    end # context
+    end # context
+  end # describe
+end # module

data/spec/spec_helper.rb

@@ -1,14 +1,16 @@
 require 'bundler/setup'
-require 'skeem'
+require 'rspec' # Use the RSpec framework
+require_relative '../lib/skeem'
 
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = '.rspec_status'
 
-  # Disable RSpec exposing methods globally on `Module` and `main`
-  config.disable_monkey_patching!
-
   config.expect_with :rspec do |c|
+    # Disable the `should` synta
     c.syntax = :expect
   end
+
+  # Display stack trace in case of failure
+  config.full_backtrace = true
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: skeem
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-24 00:00:00.000000000 Z
+date: 2018-08-25 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rley
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -71,8 +85,11 @@ files:
 - Rakefile
 - appveyor.yml
 - lib/skeem.rb
+- lib/skeem/stoken.rb
+- lib/skeem/tokenizer.rb
 - lib/skeem/version.rb
 - skeem.gemspec
+- spec/skeem/tokenizer_spec.rb
 - spec/skeem_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/famished-tiger/Skeem
@@ -102,4 +119,5 @@ specification_version: 4
 summary: Skeem is an interpreter of a subset of the Scheme programming language. Scheme
   is a descendent of the Lisp language.
 test_files:
+- spec/skeem/tokenizer_spec.rb
 - spec/skeem_spec.rb