influxdb-lineprotocol-parser 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8e81800340bcf80271e56f5c19d03ea4cd8b9535e8d0fd5bd86e605f3dae124
-  data.tar.gz: 1baa9ee265fc57e8b7bf987b8314c9305c8ab3f0240066ea9db2d901b425a35d
+  metadata.gz: a8e433f4f34e310f1886a551df2e5673f8f3ca4831f786cb22bb490b31740dc5
+  data.tar.gz: 5fd20ecf8b09f2fa682fff68dad78fdecd6c47c87f9cbe0d887ba57d4978565b
 SHA512:
-  metadata.gz: 3655611e6edbfce8c960707bf72f1f3972a7d8ced12488280a088646145551e5668329766143df26f4258acc17ffa17233e7ea9f802240d4a579d01b99e4b55f
-  data.tar.gz: ba6b7f5b37fcdc32f4e46a51d4fb7ed92b34921cb0c173ba80664a838dafef33008423745b21193592f483161edb66e916116dd353d7234f53d4db4932e05f2d
+  metadata.gz: e0273635beaf425e9d284219ac2e513bb3bc467fd10f65e6e9600e368743bfa26ac5e22edba50a50778e20313355dd3a1c79b9ecc6e89b8157626c3db3efe627
+  data.tar.gz: c8e24fde968761c85a6eecaa6d6a51bde1976a950b167bed45b53636e70644d5a18f9394bf857473d813df9bfa7915bd2f62a00d8e4f8257ec1173ce32b453cf

data/lib/influxdb/lineprotocol/parser.rb

@@ -17,7 +17,7 @@ require 'logger'
 ##
 # Extension to InfluxDB module.
 #
-module InfluxDBExt
+module InfluxDB
   ##
   # Line Protocol module.
   #
@@ -27,9 +27,15 @@ module InfluxDBExt
     # Line Protocol parser.
     #
     class Parser
-      def initialize(logger: nil, log_level: :warn)
+      def initialize(logger: nil, log_level: :warn, escapes: nil)
         @log = logger || ::Logger.new(STDERR)
         @log.level = log_level
+        case escapes
+        when :compat
+          @unescapes = InfluxDB::LineProtocol::CompatUnescapes.new
+        else
+          @unescapes = InfluxDB::LineProtocol::Unescapes.new
+        end
         enter_initial
       end
 
@@ -40,9 +46,11 @@ module InfluxDBExt
       #
       # If block is not given, returns a list of points in data.
       #
+      # The data can be a String, or a single Integer or an Array of Integers.
+      # The Integers are assumed to be UTF-8 bytes.
+      #
       def each_point(data)
-        # TODO: support byte arrays (client does not have to ensure that only complete UTF-8 characters are passed)
-        buf = data.to_s.encode(UTF_8).bytes.freeze
+        buf = bytes(data)
         i = 0
         len = buf.size
 
@@ -82,14 +90,14 @@ module InfluxDBExt
       BACKSLASH = 92
       COMMA = 44
       EQUALS = 61
-      HASH = 23
+      HASH = 35
       NEWLINE = 10
       NULL = 0
       SPACE = 32
       TAB = 9
 
       # Start (and end) marker of a string field value (not special anywhere else)
-      QUOTATION_MARK = 22
+      QUOTATION_MARK = 34
 
       # Start markers of a numeric field value (not special anywhere else)
       PLUS_SIGN = 43
@@ -117,7 +125,7 @@ module InfluxDBExt
         when COMMA
           @log.error "initial: missing measurement"
           @state = :invalid
-          invalid(buf, i, len)
+          i + 1
         when HASH # comment
           @state = :comment
           i + 1
@@ -139,21 +147,21 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when COMMA # start of tag set.
-              @point = {series: decode(buf[start..i-1]), tags: {}}
+              @point = {series: decode(buf, start, i-1), tags: {}, values: {}}
               @state = :tag_key
               return i+1
             when NEWLINE
               @log.error("measurement: missing fields")
-              # no need to go via :invalid; already at newline
-              enter_initial
-              return i+1
+              @state = :invalid
+              return i
             when SPACE # start of field set
-              @point = {series: decode(buf[start..i-1]), values: {}}
+              @point = {series: decode(buf, start, i-1), values: {}}
               @state = :field_key
               i, _ = whitespace(buf, i + 1, len)
               return i
@@ -176,23 +184,24 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when EQUALS
-              @key = decode(buf[start..i-1])
+              @key = decode(buf, start, i-1)
               if @key == ""
                 @log.error("tag_key: empty key")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @state = :tag_value
               return i+1
             when NEWLINE
               @log.error("tag key: newline")
-              enter_initial
-              return i+1
+              @state = :invalid
+              return i
             else
               i += 1
             end
@@ -212,21 +221,22 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when COMMA
-              @point[:tags][@key] = decode(buf[start..i-1])
+              @point[:tags][@key] = decode(buf, start, i-1)
               @key = nil
               @state = :tag_key
               return i+1
             when NEWLINE
               @log.error("tag value: newline")
-              enter_initial
-              return i+1
+              @state = :invalid
+              return i
             when SPACE
-              @point[:tags][@key] = decode(buf[start..i-1])
+              @point[:tags][@key] = decode(buf, start, i-1)
               @key = nil
               @state = :field_key
               i, _ = whitespace(buf, i + 1, len)
@@ -250,23 +260,24 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when EQUALS
-              @key = decode(buf[start..i-1])
+              @key = decode(buf, start, i-1)
               if @key == ""
                 @log.error("field key: empty key")
                 @state = :invalid
-                return invalid(buf, i + 1, len)
+                return i
               end
               @state = :field_value
               return i+1
             when NEWLINE
               @log.error("field key: newline")
-              enter_initial
-              return i+1
+              @state = :invalid
+              return i
             else
               i += 1
             end
@@ -283,6 +294,7 @@ module InfluxDBExt
           return len
         end
         c = buf[i]
+        raise "unsupported input type" unless c.is_a? Integer
         case c
         when LATIN_CAPITAL_LETTER_F, LATIN_CAPITAL_LETTER_T, LATIN_SMALL_LETTER_F, LATIN_SMALL_LETTER_T
           @state = :field_value_boolean
@@ -296,7 +308,7 @@ module InfluxDBExt
         else
           @log.error("field value: invalid")
           @state = :invalid
-          invalid(buf, i, len)
+          i
         end
       end
 
@@ -308,23 +320,24 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when COMMA
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value boolean: invalid boolean")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @point[:values][@key] = value
               @key = nil
               @state = :field_key
               return i+1
             when NEWLINE
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value boolean: invalid boolean")
                 enter_initial
@@ -335,11 +348,11 @@ module InfluxDBExt
               @state = :complete
               return i+1
             when SPACE
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value boolean: invalid boolean")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @point[:values][@key] = value
               @key = nil
@@ -365,34 +378,35 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when COMMA
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value numeric: invalid number")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @point[:values][@key] = value
               @key = nil
               @state = :field_key
               return i+1
             when NEWLINE
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value numeric: invalid number")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @point[:values][@key] = value
               @key = nil
               @state = :complete
               return i+1
             when SPACE
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value numeric: invalid number")
                 @state = :invalid
@@ -422,16 +436,17 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when QUOTATION_MARK
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("field value string: invalid string")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @point[:values][@key] = value
               @key = nil
@@ -450,7 +465,9 @@ module InfluxDBExt
 
       def field_value_string_end(buf, i, len)
         if i < len
-          case buf[i]
+          c = buf[i]
+          raise "unsupported input type" unless c.is_a? Integer
+          case c
           when COMMA
             @state = :field_key
             i + 1
@@ -463,7 +480,7 @@ module InfluxDBExt
             i
           else
             @state = :invalid
-            invalid(buf, i, len)
+            i
           end
         else
           len
@@ -478,16 +495,17 @@ module InfluxDBExt
             i += 1
           else
             c = buf[i]
+            raise "unsupported input type" unless c.is_a? Integer
             case c
             when BACKSLASH
               @escaped = true
               i += 1
             when NEWLINE
-              value = decode(buf[start..i-1])
+              value = decode(buf, start, i-1)
               if value.nil?
                 @log.error("timestamp: invalid timestamp")
                 @state = :invalid
-                return invalid(buf, i, len)
+                return i
               end
               @point[:timestamp] = value
               @key = nil
@@ -508,6 +526,7 @@ module InfluxDBExt
         i = line_end(buf, i, len)
         if i < len
           enter_initial
+          i += 1
         end
         i
       end
@@ -516,18 +535,19 @@ module InfluxDBExt
         i = line_end(buf, i, len)
         if i < len
           enter_initial
+          i += 1
         end
         i
       end
 
       # Starting from position i,
-      # returns the index of the byte that follows next newline.
-      # Returns len if no such byte is found or it is at the end of buf.
+      # returns the index of the newline.
+      # Returns len if no such byte is found.
       def line_end(buf, i, len)
         while i < len
           c = buf[i]
+          raise "unsupported input type" unless c.is_a? Integer
           if c == NEWLINE
-            i += 1
             return i
           end
           i += 1
@@ -541,6 +561,7 @@ module InfluxDBExt
       def whitespace(buf, i, len)
         while i < len
           c = buf[i]
+          raise "unsupported input type" unless c.is_a? Integer
           if c != SPACE && c != TAB && c != NULL
             return [i, c]
           end
@@ -549,12 +570,16 @@ module InfluxDBExt
         [len, nil]
       end
 
-      def decode(buf)
-        str = @buf.nil? ? string(buf) : string(@buf + buf)
+      def decode(buf, start, i)
+        str = if @buf.nil?
+                (start <= i) ? string(buf[start..i]) : ""
+              else
+                (start <= i) ? string(@buf + buf[start..i]) : string(@buf)
+              end
         @buf = nil
         case @state
         when :measurement
-          str
+          @unescapes.unescape(:measurement, str)
         when :tag_key
           str
         when :tag_value
@@ -595,9 +620,48 @@ module InfluxDBExt
         end
       end
 
+      def bytes(data)
+        case data
+        when nil
+          [].freeze
+        when Integer
+          [data].freeze
+        when String
+          data.encode(UTF_8).bytes.freeze
+        when Array
+          data
+        end
+      end
+
       def string(buf)
         buf.pack(UTF_8_PACK_FORMAT).force_encoding(UTF_8)
       end
+    end # Parser
+
+    class CompatUnescapes
+      def unescape(field, value)
+        case field
+        when :measurement
+          value.gsub(/\\([, ])/, '\\1')
+          # escaped comma anywhere
+          # escaped space anywhere
+        end
+      end
+    end
+
+    class Unescapes
+      def unescape(field, value)
+        case field
+        when :measurement
+          # 1. escaped hash, null, or tab at the beginning
+          # 2. escaped comma, space, or newline anywhere
+          # 3. escaped backslash at the end
+          value
+              .sub(/^\\([#\0\t])/, '\\1')
+              .gsub(/\\([, \n])/, '\\1')
+              .sub(/\\\\$/, '\\')
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: influxdb-lineprotocol-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Mikko Värri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-15 00:00:00.000000000 Z
+date: 2019-05-16 00:00:00.000000000 Z
 dependencies: []
 description: 'Streaming parser for InfluxDB line protocol.