sie 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b28813c3d6a7133a25a4addc2a52c800b14039ca
-  data.tar.gz: 6c9dce2f745c1c95e4151c43e8aa6b7428ca5ce6
+  metadata.gz: 82618d146dcca7af3a358baa84ebabc1349d4e20
+  data.tar.gz: cbaa13f1ca6a9eaeb7aa043635df08c25e7f127f
 SHA512:
-  metadata.gz: 63ff773eb13b891e0b88ef721b1f96d8befaf02a5bd62fce1b60233e2f850dcae1ccf01ce4b0e6e629463f374bee7c215595811b19a5b70a2dbae7f5fee9075a
-  data.tar.gz: 1450f07a32f74d87dad7e1989082a04207393e545d7325c5ca1ba829f39462724020c16793f1227e8dad30b9f9d1fb4f72d93bcf645aa24bb66c3b67e123d5e5
+  metadata.gz: 0eb13e5a90a1437c06454f43a3e4d4397aa6010487241fe803c92e73fc9fc32fde59054ce762cfc73e09c1c88433eec048fe6d5851887f7c076cd7fb8bfe19ac
+  data.tar.gz: e1bbfe37c0f369312b63bc55fcc9a9efe69566e8bb514384b4790a9743105eeab4aeb105343964510d97bc46130b07b21c561ab1d2a164ff58eda5d12e148635

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.idea

data/lib/sie/parser.rb

@@ -3,10 +3,10 @@ require "sie/parser/line_parser"
 
 module Sie
   class Parser
-    # TODO: Could this format knowledge be centrailized somewhere, some
-    # of this is duplicated in Character.
     BEGINNING_OF_ARRAY = "{"
     END_OF_ARRAY       = "}"
+
+    # TODO: Could this format knowledge be shared with Tokenizer? It's slightly different there.
     ENTRY              = /^\s*#/
 
     attr_private :options

data/lib/sie/parser/tokenizer.rb

@@ -1,6 +1,5 @@
 require "strscan"
 require "sie/parser/tokenizer/token"
-require "sie/parser/tokenizer/character"
 
 module Sie
   class Parser
@@ -8,22 +7,26 @@ module Sie
       pattr_initialize :line
 
       def tokenize
-        @tokens = []
-        @consume = false
-        @quoted = false
+        tokens = []
+        check_for_control_characters
 
         loop do
-          move_to_next_character
-          break unless current_character.value
-
-          if consume?
-            if quoted?
-              consume_quoted_value
-            else
-              consume_unquoted_value
-            end
+          case
+          when whitespace?
+            next
+          when match = find_entry
+            tokens << EntryToken.new(match)
+          when begin_array?
+            tokens << BeginArrayToken.new
+          when end_array?
+            tokens << EndArrayToken.new
+          when match = find_string
+            tokens << StringToken.new(match)
+          when end_of_string?
+            break
           else
-            add_new_token
+            # We shouldn't get here, but if we do we need to bail out, otherwise we get an infinite loop.
+            fail "Unhandled character in line at position #{scanner.pos}: " + scanner.string
           end
         end
 
@@ -32,55 +35,64 @@ module Sie
 
       private
 
-      attr_query :consume?, :quoted?
-      attr_private :consume, :quoted, :tokens, :current_character
+      def check_for_control_characters
+        if /(.*?)([\x00-\x08\x0a-\x1f\x7f])/.match(line)
+          fail "Unhandled character in line at position #{$1.length + 1}: " + scanner.string
+        end
+      end
 
-      def move_to_next_character
-        @current_character = Character.new(scanner.getch)
+      def whitespace?
+        scanner.scan(/[ \t]+/)
       end
 
-      def consume_quoted_value
-        if current_character.quote?
-          @quoted = false
-          @consume = false
+      def find_entry
+        match = scanner.scan(/#\S+/)
+
+        if match
+          match.sub(/\A#/, "")
         else
-          add_to_current_token current_character
+          nil
         end
       end
 
-      def consume_unquoted_value
-        if current_character.unquoted_data?
-          add_to_current_token current_character
+      def begin_array?
+        scanner.scan(/#{Sie::Parser::BEGINNING_OF_ARRAY}/)
+      end
+
+      def end_array?
+        scanner.scan(/#{Sie::Parser::END_OF_ARRAY}/)
+      end
+
+      def find_string
+        match = find_quoted_string || find_unquoted_string
+
+        if match
+          remove_unnessesary_escapes(match)
         else
-          @consume = false
+          nil
         end
       end
 
-      def add_new_token
-        if current_character.entry?
-          @consume = true
-          add_token EntryToken.new
-        elsif current_character.beginning_of_array?
-          add_token BeginArrayToken.new
-        elsif current_character.end_of_array?
-          add_token EndArrayToken.new
-        elsif current_character.quote?
-          @consume = @quoted = true
-          add_token StringToken.new
-        elsif current_character.non_whitespace?
-          @consume = true
-          add_token StringToken.new(current_character.value)
-        elsif current_character.value != " "
-          raise "Unhandled character: #{current_character.value}"
+      def end_of_string?
+        scanner.eos?
+      end
+
+      def find_quoted_string
+        match = scanner.scan(/"(\\"|[^"])*"/)
+
+        if match
+          match.sub(/\A"/, "").sub(/"\z/, "")
+        else
+          nil
         end
       end
 
-      def add_token(token)
-        tokens << token
+      def find_unquoted_string
+        scanner.scan(/\S+/)
       end
 
-      def add_to_current_token(character)
-        tokens.last.value += character.value
+      def remove_unnessesary_escapes(match)
+        match.gsub(/\\([\\"])/, "\\1")
       end
 
       def scanner

data/lib/sie/version.rb

@@ -1,4 +1,4 @@
 module Sie
   # For versioning see: http://semver.org/
-  VERSION = "3.1.1"
+  VERSION = "3.2.0"
 end

data/spec/unit/parser/tokenizer_spec.rb

@@ -34,6 +34,62 @@ describe Sie::Parser::Tokenizer do
     ])
   end
 
+  it "handles escaped quotes in quoted strings" do
+    tokenizer = Sie::Parser::Tokenizer.new('"String with \\" quote"')
+    tokens = tokenizer.tokenize
+
+    expect(token_table_for(tokens)).to eq([
+                                              [ "StringToken", 'String with " quote']
+                                          ])
+  end
+
+  it "handles escaped quotes in non-quoted strings" do
+    tokenizer = Sie::Parser::Tokenizer.new('String_with_\\"_quote')
+    tokens = tokenizer.tokenize
+
+    expect(token_table_for(tokens)).to eq([
+                                              [ "StringToken", 'String_with_"_quote']
+                                          ])
+  end
+
+  it "handles escaped backslash in strings" do
+    tokenizer = Sie::Parser::Tokenizer.new('"String with \\\\ backslash"')
+    tokens = tokenizer.tokenize
+
+    expect(token_table_for(tokens)).to eq([
+                                              [ "StringToken", 'String with \\ backslash']
+                                          ])
+  end
+
+  it "has reasonable behavior for consecutive escape characters" do
+    tokenizer = Sie::Parser::Tokenizer.new('"\\\\\\"\\\\"')
+    tokens = tokenizer.tokenize
+
+    expect(token_table_for(tokens)).to eq([
+                                              [ "StringToken", '\\"\\']
+                                          ])
+  end
+
+  it "handles tab character as field separator" do
+    tokenizer = Sie::Parser::Tokenizer.new("#TRANS\t2400")
+    tokens = tokenizer.tokenize
+
+    expect(token_table_for(tokens)).to eq([
+                                              [ "EntryToken", "TRANS"],
+                                              [ "StringToken", "2400"]
+                                          ])
+  end
+
+  it "rejects control characters" do
+    codes_not_allowed = (0..8).to_a + (10..31).to_a + [127]
+    codes_not_allowed.each do |x|
+      tokenizer = Sie::Parser::Tokenizer.new([x].pack("C"))
+      expect{tokenizer.tokenize}.to raise_error /Unhandled character/
+    end
+  end
+
+  private
+
   def token_table_for(tokens)
     tokens.map { |token|
       [ token.class.name.split("::").last, token.value ]

data/spec/unit/parser_spec.rb

@@ -3,21 +3,40 @@ require "sie/parser"
 
 describe Sie::Parser, "parse" do
   it "parses sie data that includes arrays" do
+    data = <<-DATA
+#VER "LF" 2222 20130101 "Foocorp expense"
+{
+#TRANS 2400 {} -200 20130101 "Foocorp expense"
+#TRANS 4100 {} 180 20130101 "Widgets from foocorp"
+#TRANS 2611 {} -20 20130101 "VAT"
+}
+    DATA
+
     parser = Sie::Parser.new
-    sie_file = parser.parse(<<DATA
+    sie_file = parser.parse(data)
+
+    voucher_entry = sie_file.entries.first
+    expect(sie_file.entries.size).to eq(1)
+    expect(voucher_entry.attributes["verdatum"]).to eq("20130101")
+    expect(voucher_entry.entries.size).to eq(3)
+    expect(voucher_entry.entries.first.attributes["kontonr"]).to eq("2400")
+  end
+
+  it "handles leading whitespace" do
+    data = <<-DATA
 #VER "LF" 2222 20130101 "Foocorp expense"
 {
     #TRANS 2400 {} -200 20130101 "Foocorp expense"
     #TRANS 4100 {} 180 20130101 "Widgets from foocorp"
     #TRANS 2611 {} -20 20130101 "VAT"
 }
-DATA
-    )
+    DATA
+
+    parser = Sie::Parser.new
+    sie_file = parser.parse(data)
 
     voucher_entry = sie_file.entries.first
     expect(sie_file.entries.size).to eq(1)
-    expect(voucher_entry.attributes["verdatum"]).to eq("20130101")
     expect(voucher_entry.entries.size).to eq(3)
-    expect(voucher_entry.entries.first.attributes["kontonr"]).to eq("2400")
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sie
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.2.0
 platform: ruby
 authors:
 - Barsoom AB
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-02 00:00:00.000000000 Z
+date: 2015-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attr_extras
@@ -105,7 +105,6 @@ files:
 - lib/sie/parser/line_parser.rb
 - lib/sie/parser/sie_file.rb
 - lib/sie/parser/tokenizer.rb
-- lib/sie/parser/tokenizer/character.rb
 - lib/sie/parser/tokenizer/token.rb
 - lib/sie/version.rb
 - script/bootstrap

data/lib/sie/parser/tokenizer/character.rb

@@ -1,34 +0,0 @@
-module Sie
-  class Parser
-    class Tokenizer
-      class Character
-        pattr_initialize :value
-        attr_reader :value
-
-        def unquoted_data?
-          non_whitespace? && !end_of_array?
-        end
-
-        def entry?
-          value == "#"
-        end
-
-        def beginning_of_array?
-          value == "{"
-        end
-
-        def end_of_array?
-          value == "}"
-        end
-
-        def quote?
-          value == '"'
-        end
-
-        def non_whitespace?
-          value != " " && value != "\t"
-        end
-      end
-    end
-  end
-end