rbscmlex 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2860b1b9f6d206dcfacf04fba88a5c64e9670c97d42423d5fac78615c3609d3a
-  data.tar.gz: c2ab02c9febc928efa76fa47aea965c27a53099b56ff55e180304988f9eea062
+  metadata.gz: 23174f34d0abd8999a84dcbe74b96fb7042143b619d028c8cb96f2e5a07b1ef2
+  data.tar.gz: 85d495b784530ae42e435ccf9c47adb274ca3eb4d635fb64b41e1843bafc4687
 SHA512:
-  metadata.gz: 8a008f3c549e69bfa839d6788449704afb5b2491084eacde00ce1809769d4c65fffc4f22b0843c5b7522908891146f4ad4f227ef1ae67f1f07d0a33f97bbba3a
-  data.tar.gz: 68e14c34c40ea2b4715a20d9cac1f56c6860eeb6d9d3c8e2d31b6addf66fb529217e88b272ef1ccc252cb52645fb41dc3d77a8c6d0eb5af902b936af7b851e87
+  metadata.gz: 984827b5d264e2b8f51b3582b3cb46ff595f1a50e965ecc67e16fd9650932107f1c7a570040faf105d2d9aed0d9463ac7c3b4f46dc7ee3b8a4f56bc99ada9589
+  data.tar.gz: a98522d875d75c8136fc4507a54ed737d461183c59e263b2a0dc5fa4034078da042e5ef70d0f9c1672fbd1c6322f938455921d42957d3085fa9df506d6580494

data/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 ## [Unreleased]
 - (nothing to record here)
 
+## [0.1.4] - 2021-05-23
+### Fixed
+- Fix issue #7: cannot tokenize a string contains any spaces.
+
 ## [0.1.3] - 2021-05-15
 ### Added
 - Add `Lexer#skip_token(offset)`

data/lib/rbscmlex.rb

@@ -5,4 +5,9 @@ module Rbscmlex
   require_relative "rbscmlex/token"
   require_relative "rbscmlex/lexer"
   require_relative "rbscmlex/version"
+
+  def self.lexer(source)
+    Lexer.new(source)
+  end
+
 end

data/lib/rbscmlex/lexer.rb

@@ -169,8 +169,6 @@ module Rbscmlex
       raise StopIteration if (@next_pos + offset) >= size
     end
 
-    # :stopdoc:
-
     S2R_MAP = { "(" => "( ", ")" => " ) ", "'" => " ' " }
 
     BOOLEAN    = /\A#(f(alse)?|t(rue)?)\Z/
@@ -197,12 +195,10 @@ module Rbscmlex
     CHAR_PAT    = "(#{SINGLE_CHAR_PAT}|#{SPACE_PAT}|#{NEWLINE_PAT})"
     CHAR        = Regexp.new("\\A#{CHAR_PREFIX}#{CHAR_PAT}\\Z")
 
-    # :startdoc:
-
     def tokenize(src)
-      cooked = src.gsub(/[()']/, S2R_MAP)
+      cooked = src.gsub(/[()']/, S2R_MAP).strip
 
-      cooked.split(" ").map { |literal|
+      split_at_space(cooked).map { |literal|
         case literal
         when "("
           Rbscmlex.new_token(:lparen, literal)
@@ -234,6 +230,44 @@ module Rbscmlex
       }
     end
 
+    def split_at_space(source)
+      escaped = false
+      in_string = false
+      after_space = false
+      slice = String.new        # an empty string
+      results = []
+      source.each_char { |rune|
+        case rune
+        when "\\"
+          slice << rune
+          escaped = !escaped if in_string
+          after_space = false
+        when '"'
+          if !in_string or !escaped
+            in_string = !in_string
+          end
+          slice << rune
+          escaped = false
+          after_space = false
+        when /[\s]/
+          if in_string
+            slice << rune
+          elsif !after_space
+            results << slice.dup
+            slice.clear
+          end
+          escaped = false
+          after_space = true
+        else
+          slice << rune
+          escaped = false
+          after_space = false
+        end
+      }
+      results << slice unless slice.empty?
+      results
+    end
+
     # Holds functions to check a literal is valid as an identifier
     # defined in R7RS.
     #

data/lib/rbscmlex/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Rbscmlex
-  VERSION = "0.1.3"
-  RELEASE = "2021-05-15"
+  VERSION = "0.1.4"
+  RELEASE = "2021-05-23"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbscmlex
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - mnbi
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-15 00:00:00.000000000 Z
+date: 2021-05-22 00:00:00.000000000 Z
 dependencies: []
 description: A simple lexical analyzer for Scheme
 email: