influxdb-lineprotocol-parser 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2490d6234974b7e68904ea3e1e0f8097d43aafd15946632222e6428bf84920b
-  data.tar.gz: 501f6bf3750299aea99024b1180841bbaee15e343cf7d9ab843726c0bfcdb5c1
+  metadata.gz: e8e81800340bcf80271e56f5c19d03ea4cd8b9535e8d0fd5bd86e605f3dae124
+  data.tar.gz: 1baa9ee265fc57e8b7bf987b8314c9305c8ab3f0240066ea9db2d901b425a35d
 SHA512:
-  metadata.gz: '094ae5413c86aab7448ca3b4b1e2a6ff90b70f8559434493e6efd02ade64cbf60e965eaf8f760e2fa00afe18e9c86880e3501c627e8aee569ba990b209d7d4b8'
-  data.tar.gz: 9f03253c475b67532099c956609ec833e5fb934c19979574ec4aafe382c8b8553b61e1bb2b533cedd20743adddcaf5f7e118675f7de914eccf7984dc6e0730bf
+  metadata.gz: 3655611e6edbfce8c960707bf72f1f3972a7d8ced12488280a088646145551e5668329766143df26f4258acc17ffa17233e7ea9f802240d4a579d01b99e4b55f
+  data.tar.gz: ba6b7f5b37fcdc32f4e46a51d4fb7ed92b34921cb0c173ba80664a838dafef33008423745b21193592f483161edb66e916116dd353d7234f53d4db4932e05f2d

data/lib/influxdb/lineprotocol/parser.rb

@@ -12,18 +12,592 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+require 'logger'
+
+##
+# Extension to InfluxDB module.
+#
 module InfluxDBExt
+  ##
+  # Line Protocol module.
+  #
   module LineProtocol
+
+    ##
+    # Line Protocol parser.
+    #
     class Parser
+      def initialize(logger: nil, log_level: :warn)
+        @log = logger || ::Logger.new(STDERR)
+        @log.level = log_level
+        enter_initial
+      end
+
+      ##
+      # Parse the points from data.
+      #
       # If block is given, yields each point in data.
+      #
       # If block is not given, returns a list of points in data.
-      def each_point data
-        if block_given?
-          yield nil
+      #
+      def each_point(data)
+        # TODO: support byte arrays (client does not have to ensure that only complete UTF-8 characters are passed)
+        buf = data.to_s.encode(UTF_8).bytes.freeze
+        i = 0
+        len = buf.size
+
+        points = block_given? ? nil : []
+
+        while i < len
+          i = self.send(@state, buf, i, len)
+          if @state == :complete
+            if block_given?
+              yield @point
+            else
+              points << @point
+            end
+            enter_initial
+          end
+        end
+
+        points
+      end
+
+      private
+
+      def enter_initial
+        @point = nil
+        @state = :initial
+        @escaped = false
+        @buf = nil
+        @key = nil
+      end
+
+      UTF_8 = Encoding.find 'UTF-8'
+      UTF_8_PACK_FORMAT = 'C*'.freeze
+
+      # All the special bytes Line Protocol handles.
+      # In UTF-8, these are all single byte characters.
+      # Any multi-byte characters are just skipped as part of the tokens (measurement, tag key, tag value, ...).
+      BACKSLASH = 92
+      COMMA = 44
+      EQUALS = 61
+      HASH = 23
+      NEWLINE = 10
+      NULL = 0
+      SPACE = 32
+      TAB = 9
+
+      # Start (and end) marker of a string field value (not special anywhere else)
+      QUOTATION_MARK = 22
+
+      # Start markers of a numeric field value (not special anywhere else)
+      PLUS_SIGN = 43
+      MINUS_SIGN = 45
+      DECIMAL_POINT = 46
+      DIGIT_ZERO = 48
+      DIGIT_NINE = 57
+
+      # Start markers of a Boolean field value (not special anywhere else)
+      LATIN_CAPITAL_LETTER_F = 70
+      LATIN_CAPITAL_LETTER_T = 84
+      LATIN_SMALL_LETTER_F = 102
+      LATIN_SMALL_LETTER_T = 116
+
+
+
+      def initial(buf, i, len)
+        # whitespace consumes TAB, SPACE, and NULL.
+        # This method consumes NEWLINE, HASH, and COMMA.
+        # BACKSLASH and EQUAL (of the special bytes) are valid measurement starts; they are not consumed.
+        i, c = whitespace(buf, i, len)
+        case c
+        when nil # just whitespace
+          len
+        when COMMA
+          @log.error "initial: missing measurement"
+          @state = :invalid
+          invalid(buf, i, len)
+        when HASH # comment
+          @state = :comment
+          i + 1
+        when NEWLINE
+          i + 1
+        else
+          # don't advance i because the byte belongs to measurement
+          @log.info "initial: start measurement at offset #{i}"
+          @state = :measurement
+          i
+        end
+      end
+
+      def measurement(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when COMMA # start of tag set.
+              @point = {series: decode(buf[start..i-1]), tags: {}}
+              @state = :tag_key
+              return i+1
+            when NEWLINE
+              @log.error("measurement: missing fields")
+              # no need to go via :invalid; already at newline
+              enter_initial
+              return i+1
+            when SPACE # start of field set
+              @point = {series: decode(buf[start..i-1]), values: {}}
+              @state = :field_key
+              i, _ = whitespace(buf, i + 1, len)
+              return i
+            else # part of measurement
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def tag_key(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when EQUALS
+              @key = decode(buf[start..i-1])
+              if @key == ""
+                @log.error("tag_key: empty key")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @state = :tag_value
+              return i+1
+            when NEWLINE
+              @log.error("tag key: newline")
+              enter_initial
+              return i+1
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def tag_value(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when COMMA
+              @point[:tags][@key] = decode(buf[start..i-1])
+              @key = nil
+              @state = :tag_key
+              return i+1
+            when NEWLINE
+              @log.error("tag value: newline")
+              enter_initial
+              return i+1
+            when SPACE
+              @point[:tags][@key] = decode(buf[start..i-1])
+              @key = nil
+              @state = :field_key
+              i, _ = whitespace(buf, i + 1, len)
+              return i
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def field_key(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when EQUALS
+              @key = decode(buf[start..i-1])
+              if @key == ""
+                @log.error("field key: empty key")
+                @state = :invalid
+                return invalid(buf, i + 1, len)
+              end
+              @state = :field_value
+              return i+1
+            when NEWLINE
+              @log.error("field key: newline")
+              enter_initial
+              return i+1
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def field_value(buf, i, len)
+        if i == len
+          return len
+        end
+        c = buf[i]
+        case c
+        when LATIN_CAPITAL_LETTER_F, LATIN_CAPITAL_LETTER_T, LATIN_SMALL_LETTER_F, LATIN_SMALL_LETTER_T
+          @state = :field_value_boolean
+          i
+        when DIGIT_ZERO..DIGIT_NINE, PLUS_SIGN, MINUS_SIGN, DECIMAL_POINT
+          @state = :field_value_numeric
+          i
+        when QUOTATION_MARK
+          @state = :field_value_string
+          i + 1
+        else
+          @log.error("field value: invalid")
+          @state = :invalid
+          invalid(buf, i, len)
+        end
+      end
+
+      def field_value_boolean(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when COMMA
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value boolean: invalid boolean")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :field_key
+              return i+1
+            when NEWLINE
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value boolean: invalid boolean")
+                enter_initial
+                return i + 1
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :complete
+              return i+1
+            when SPACE
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value boolean: invalid boolean")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :timestamp
+              i, _ = whitespace(buf, i + 1, len)
+              return i
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def field_value_numeric(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when COMMA
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value numeric: invalid number")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :field_key
+              return i+1
+            when NEWLINE
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value numeric: invalid number")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :complete
+              return i+1
+            when SPACE
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value numeric: invalid number")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :timestamp
+              i, _ = whitespace(buf, i + 1, len)
+              return i
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def field_value_string(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when QUOTATION_MARK
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("field value string: invalid string")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:values][@key] = value
+              @key = nil
+              @state = :field_value_string_end
+              return i+1
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def field_value_string_end(buf, i, len)
+        if i < len
+          case buf[i]
+          when COMMA
+            @state = :field_key
+            i + 1
+          when NEWLINE
+            @state = :complete
+            i + 1
+          when SPACE
+            @state = :timestamp
+            i, _ = whitespace(buf, i, len)
+            i
+          else
+            @state = :invalid
+            invalid(buf, i, len)
+          end
+        else
+          len
+        end
+      end
+
+      def timestamp(buf, i, len)
+        start = i
+        while i < len
+          if @escaped
+            @escaped = false
+            i += 1
+          else
+            c = buf[i]
+            case c
+            when BACKSLASH
+              @escaped = true
+              i += 1
+            when NEWLINE
+              value = decode(buf[start..i-1])
+              if value.nil?
+                @log.error("timestamp: invalid timestamp")
+                @state = :invalid
+                return invalid(buf, i, len)
+              end
+              @point[:timestamp] = value
+              @key = nil
+              @state = :complete
+              return i + 1
+            else
+              i += 1
+            end
+          end
+        end
+        if i == len && start < i
+          @buf.nil? ? @buf = buf[start..i-1] : @buf += buf[start..i-1]
+        end
+        i
+      end
+
+      def comment(buf, i, len)
+        i = line_end(buf, i, len)
+        if i < len
+          enter_initial
+        end
+        i
+      end
+
+      def invalid(buf, i, len)
+        i = line_end(buf, i, len)
+        if i < len
+          enter_initial
+        end
+        i
+      end
+
+      # Starting from position i,
+      # returns the index of the byte that follows next newline.
+      # Returns len if no such byte is found or it is at the end of buf.
+      def line_end(buf, i, len)
+        while i < len
+          c = buf[i]
+          if c == NEWLINE
+            i += 1
+            return i
+          end
+          i += 1
+        end
+        len
+      end
+
+      # Starting from position i,
+      # return index of the first non-whitespace byte and the byte itself.
+      # Return len and nil if no such byte is found.
+      def whitespace(buf, i, len)
+        while i < len
+          c = buf[i]
+          if c != SPACE && c != TAB && c != NULL
+            return [i, c]
+          end
+          i += 1
+        end
+        [len, nil]
+      end
+
+      def decode(buf)
+        str = @buf.nil? ? string(buf) : string(@buf + buf)
+        @buf = nil
+        case @state
+        when :measurement
+          str
+        when :tag_key
+          str
+        when :tag_value
+          str
+        when :field_key
+          str
+        when :field_value_boolean
+          case str
+          when 't', 'T', 'true', 'True'
+            true
+          when 'f', 'F', 'false', 'False'
+            false
+          else
+            @log.error("invalid Boolean: #{str}")
+            nil
+          end
+        when :field_value_numeric
+          case str
+          when /^[+-]?[0-9]*\.[0-9]*([eE][+-]?[0-9]+)?$/
+            str.to_f
+          when /^[+-]?[0-9]+[ui]$/
+            str.to_i
+          else
+            @log.error("invalid number: #{str}")
+            nil
+          end
+        when :field_value_string
+          str
+        when :timestamp
+          case str
+          when /^-?[0-9]+$/
+            str.to_i
+          else
+            @log.error("invalid timestamp: #{str}")
+          end
         else
-          []
+          raise "error: decode: invalid state"
         end
       end
+
+      def string(buf)
+        buf.pack(UTF_8_PACK_FORMAT).force_encoding(UTF_8)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: influxdb-lineprotocol-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Mikko Värri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-09 00:00:00.000000000 Z
+date: 2019-05-15 00:00:00.000000000 Z
 dependencies: []
 description: 'Streaming parser for InfluxDB line protocol.
 
@@ -33,7 +33,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="