tablestore-ruby-sdk 0.0.0

data/lib/tablestore/plain_buffer_coded_input_stream.rb

@@ -0,0 +1,217 @@
+require 'consts'
+require 'tablestore/plain_buffer_crc8'
+require 'tablestore/error'
+
+class PlainBufferCodedInputStream
+  def initialize(input_stream)
+    @input_stream = input_stream
+  end
+
+  def read_tag
+    @input_stream.read_tag
+  end
+
+  def check_last_tag_was(tag)
+    @input_stream.check_last_tag_was(tag)
+  end
+
+  def get_last_tag
+    @input_stream.get_last_tag
+  end
+
+  def read_header
+    @input_stream.read_int32
+  end
+
+  def read_primary_key_value(cell_check_sum)
+    raise TableStoreClientError.new("Expect TAG_CELL_VALUE but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL_VALUE)
+    @input_stream.read_raw_little_endian32
+    column_type = @input_stream.read_raw_byte.ord
+    if column_type == VT_INTEGER
+      int64_value = @input_stream.read_int64
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_INTEGER)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, int64_value)
+      read_tag
+      return int64_value, cell_check_sum
+    elsif column_type == VT_STRING
+      value_size = @input_stream.read_int32
+      string_value = @input_stream.read_utf_string(value_size)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_STRING)
+      cell_check_sum = PlainBufferCrc8.crc_int32(cell_check_sum, value_size)
+      cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, string_value)
+      read_tag
+      return string_value, cell_check_sum
+    elsif column_type == VT_BLOB
+      value_size = @input_stream.read_int32
+      binary_value = @input_stream.read_bytes(value_size)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_BLOB)
+      cell_check_sum = PlainBufferCrc8.crc_int32(cell_check_sum, value_size)
+      cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, binary_value)
+      read_tag
+      return [binary_value], cell_check_sum
+    else
+      raise TableStoreClientError.new("Unsupported primary key type:" + column_type.to_s)
+    end
+  end
+
+  def read_column_value(cell_check_sum)
+    raise TableStoreClientError.new("Expect TAG_CELL_VALUE but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL_VALUE)
+    @input_stream.read_raw_little_endian32
+    column_type = @input_stream.read_raw_byte.ord
+    if column_type == VT_INTEGER
+      int64_value = @input_stream.read_int64
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_INTEGER)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, int64_value)
+      read_tag
+      return int64_value, cell_check_sum
+    elsif column_type == VT_STRING
+      value_size = @input_stream.read_int32
+      string_value = @input_stream.read_utf_string(value_size)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_STRING)
+      cell_check_sum = PlainBufferCrc8.crc_int32(cell_check_sum, value_size)
+      cell_check_sum =PlainBufferCrc8.crc_string(cell_check_sum, string_value)
+      read_tag
+      return string_value, cell_check_sum
+    elsif column_type == VT_BLOB
+      value_size = @input_stream.read_int32
+      binary_value = @input_stream.read_bytes(value_size)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_BLOB)
+      cell_check_sum = PlainBufferCrc8.crc_int32(cell_check_sum, value_size)
+      cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, binary_value)
+      read_tag
+      return binary_value, cell_check_sum
+    elsif column_type == VT_BOOLEAN
+      bool_value = @input_stream.read_boolean
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_BOOLEAN)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, bool_value ? 1 : 0)
+      read_tag
+      return bool_value, cell_check_sum
+    elsif column_type == VT_DOUBLE
+      double_int = @input_stream.read_int64
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_DOUBLE)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, double_int)
+      read_tag
+
+      if SYS_BITS == 64
+          double_value, = [double_int].pack('q').unpack('d')
+      else
+          double_value, = [double_int].pack('l').unpack('d')
+      end
+      return double_value, cell_check_sum
+    else
+      raise TableStoreClientError.new("Unsupported column type: " + column_type.str)
+    end
+  end
+
+  def read_primary_key_column(row_check_sum)
+    raise TableStoreClientError.new("Expect TAG_CELL but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL)
+    read_tag
+    raise TableStoreClientError.new("Expect TAG_CELL_NAME but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL_NAME)
+    cell_check_sum = 0
+    name_size = @input_stream.read_raw_little_endian32
+    column_name = @input_stream.read_utf_string(name_size)
+    cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, column_name)
+    read_tag
+    raise TableStoreClientError.new("Expect TAG_CELL_VALUE but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL_VALUE)
+    primary_key_value, cell_check_sum = read_primary_key_value(cell_check_sum)
+    if @input_stream.get_last_tag == TAG_CELL_CHECKSUM
+      check_sum = @input_stream.read_raw_byte.ord
+      raise TableStoreClientError.new("Checksum mismatch. expected:" + check_sum.to_s + ",actual:" + cell_check_sum.to_s) if check_sum != cell_check_sum
+      read_tag
+    else
+      raise TableStoreClientError.new("Expect TAG_CELL_CHECKSUM but it was " + get_last_tag.to_s)
+    end
+    row_check_sum = PlainBufferCrc8.crc_int8(row_check_sum, cell_check_sum)
+    return column_name, primary_key_value, row_check_sum
+  end
+
+  def read_column(row_check_sum)
+    raise TableStoreClientError.new("Expect TAG_CELL but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL)
+    read_tag
+    raise TableStoreClientError.new("Expect TAG_CELL_NAME but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_CELL_NAME)
+
+    cell_check_sum = 0
+    column_value = nil
+    timestamp = nil
+    name_size = @input_stream.read_raw_little_endian32
+    column_name = @input_stream.read_utf_string(name_size)
+    cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, column_name)
+    read_tag
+    if get_last_tag == TAG_CELL_VALUE
+      column_value, cell_check_sum = read_column_value(cell_check_sum)
+    end
+    # skip CELL_TYPE
+    if get_last_tag == TAG_CELL_TYPE
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, cell_type)
+      read_tag
+    end
+    if get_last_tag == TAG_CELL_TIMESTAMP
+      timestamp = @input_stream.read_int64
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, timestamp)
+      read_tag
+    end
+    if get_last_tag == TAG_CELL_CHECKSUM
+      check_sum = @input_stream.read_raw_byte.ord
+      raise TableStoreClientError.new("Checksum mismatch. expected:" + check_sum.to_s + ",actual:" + cell_check_sum.to_s) if check_sum != cell_check_sum
+      read_tag
+    else
+      raise TableStoreClientError.new("Expect TAG_CELL_CHECKSUM but it was " + get_last_tag.to_s)
+    end
+    row_check_sum = PlainBufferCrc8.crc_int8(row_check_sum, cell_check_sum)
+    return column_name, column_value, timestamp, row_check_sum
+  end
+
+  def read_row_without_header
+    row_check_sum = 0
+    primary_key = {}
+    attributes = {}
+
+    raise TableStoreClientError.new("Expect TAG_ROW_PK but it was " + get_last_tag.to_s) unless check_last_tag_was(TAG_ROW_PK)
+
+    read_tag
+    while check_last_tag_was(TAG_CELL)
+      name, value, row_check_sum = read_primary_key_column(row_check_sum)
+      primary_key[name] = value
+    end
+    if check_last_tag_was(TAG_ROW_DATA)
+      read_tag
+      while check_last_tag_was(TAG_CELL)
+        column_name, column_value, timestamp, row_check_sum = read_column(row_check_sum)
+        attributes[column_name] = column_value
+      end
+    end
+    if check_last_tag_was(TAG_DELETE_ROW_MARKER)
+      read_tag
+      row_check_sum = PlainBufferCrc8.crc_int8(row_check_sum, 1)
+    else
+      row_check_sum = PlainBufferCrc8.crc_int8(row_check_sum, 0)
+    end
+
+    if check_last_tag_was(TAG_ROW_CHECKSUM)
+      check_sum = @input_stream.read_raw_byte.ord
+      raise TableStoreClientError.new("Checksum is mismatch.") if check_sum != row_check_sum
+      read_tag
+    else
+      raise TableStoreClientError.new("Expect TAG_ROW_CHECKSUM but it was " + get_last_tag.to_s)
+    end
+    return primary_key, attributes
+  end
+
+  def read_row
+    raise TableStoreClientError.new("Invalid header from plain buffer.") if read_header != HEADER
+    read_tag
+    read_row_without_header
+  end
+
+  def read_rows
+    raise TableStoreClientError("Invalid header from plain buffer.") if read_header != HEADER
+    read_tag
+    row_list = []
+    while !@input_stream.is_at_end?
+      pk, attr = read_row_without_header
+      row_list << {pk:pk, attr: attr}
+    end
+    row_list
+  end
+
+end

data/lib/tablestore/plain_buffer_coded_output_stream.rb

@@ -0,0 +1,252 @@
+require 'consts'
+require 'tablestore/plain_buffer_crc8'
+require 'tablestore/error'
+
+class  PlainBufferCodedOutputStream
+
+  def initialize(output_stream)
+    @output_stream = output_stream
+  end
+
+  def write_header
+    @output_stream.write_raw_little_endian32(HEADER)
+  end
+
+  def write_tag(tag)
+    @output_stream.write_raw_byte(tag)
+  end
+
+  def write_cell_name(name, cell_check_sum)
+    write_tag(TAG_CELL_NAME)
+    @output_stream.write_raw_little_endian32(name.length)
+    @output_stream.write_bytes(name)
+    cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, name)
+    cell_check_sum
+  end
+
+  def write_primary_key_value(value, cell_check_sum)
+    write_tag(TAG_CELL_VALUE)
+    value = value.to_s.split("::").last if [Metadata::INF_MAX, Metadata::INF_MIN, Metadata::PK_AUTO_INCR].include?value
+    if value == "INF_MIN"
+      @output_stream.write_raw_little_endian32(1)
+      @output_stream.write_raw_byte(VT_INF_MIN)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_INF_MIN)
+    elsif value == "INF_MAX"
+      @output_stream.write_raw_little_endian32(1)
+      @output_stream.write_raw_byte(VT_INF_MAX)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_INF_MAX)
+    elsif value == "PK_AUTO_INCR"
+      @output_stream.write_raw_little_endian32(1)
+      @output_stream.write_raw_byte(VT_AUTO_INCREMENT)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_AUTO_INCREMENT)
+    elsif value.is_a?(Fixnum)
+      @output_stream.write_raw_little_endian32(1 + LITTLE_ENDIAN_64_SIZE)
+      @output_stream.write_raw_byte(VT_INTEGER)
+      @output_stream.write_raw_little_endian64(value)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_INTEGER)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, value)
+    elsif value.is_a?(String)
+      prefix_length = LITTLE_ENDIAN_32_SIZE + 1
+      @output_stream.write_raw_little_endian32(prefix_length + value.length)
+      @output_stream.write_raw_byte(VT_STRING)
+      @output_stream.write_raw_little_endian32(value.length)
+      @output_stream.write_bytes(value)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_STRING)
+      cell_check_sum = PlainBufferCrc8.crc_int32(cell_check_sum, value.length)
+      cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, value)
+    # elsif value.is_a?()
+    else
+      raise TableStoreClientError.new("Unsupported primary key type: " + value.class)
+    end
+    cell_check_sum
+  end
+
+  def write_column_value_with_checksum(value, cell_check_sum)
+    write_tag(TAG_CELL_VALUE)
+    if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+      @output_stream.write_raw_little_endian32(2)
+      @output_stream.write_raw_byte(VT_BOOLEAN)
+      @output_stream.write_boolean(value)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_BOOLEAN)
+      if value
+        cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, 1)
+      else
+        cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, 0)
+      end
+    elsif value.is_a?(Fixnum)
+      @output_stream.write_raw_little_endian32(1 + LITTLE_ENDIAN_64_SIZE)
+      @output_stream.write_raw_byte(VT_INTEGER)
+      @output_stream.write_raw_little_endian64(value)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_INTEGER)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, value)
+    elsif value.is_a?(String)
+      prefix_length = LITTLE_ENDIAN_32_SIZE + 1
+      @output_stream.write_raw_little_endian32(prefix_length + value.length)
+      @output_stream.write_raw_byte(VT_STRING)
+      @output_stream.write_raw_little_endian32(value.length)
+      @output_stream.write_bytes(value)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_STRING)
+      cell_check_sum = PlainBufferCrc8.crc_int32(cell_check_sum, value.length)
+      cell_check_sum = PlainBufferCrc8.crc_string(cell_check_sum, value)
+    elsif value.is_a?(Float)
+      if SYS_BITS == 64
+        double_in_long, = [value].pack("d").unpack("q")
+      else
+        double_in_long, = [value].pack("d").unpack("l")
+      end
+      @output_stream.write_raw_little_endian32(1 + LITTLE_ENDIAN_64_SIZE)
+      @output_stream.write_raw_byte(VT_DOUBLE)
+      @output_stream.write_double(value)
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, VT_DOUBLE)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, double_in_long)
+    else
+      raise TableStoreClientError.new("Unsupported column type: " + value.class)
+    end
+    cell_check_sum
+  end
+
+  def write_column_value(value)
+    if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+      @output_stream.write_raw_byte(VT_BOOLEAN)
+    elsif value.is_a?(Fixnum)
+      @output_stream.write_raw_byte(VT_INTEGER)
+      @output_stream.write_raw_little_endian64(value)
+    elsif value.is_a?(String)
+      @output_stream.write_raw_byte(VT_STRING)
+      @output_stream.write_raw_little_endian32(value.length)
+      @output_stream.write_bytes(value)
+    elsif value.is_a?(Float)
+      @output_stream.write_raw_byte(VT_DOUBLE)
+      @output_stream.write_double(value)
+    else
+      raise TableStoreClientError.new("Unsupported column type: " + value.class)
+    end
+  end
+
+  def write_primary_key_column(pk_name, pk_value, row_check_sum)
+    cell_check_sum = 0
+    write_tag(TAG_CELL)
+    cell_check_sum = write_cell_name(pk_name, cell_check_sum)
+    cell_check_sum = write_primary_key_value(pk_value, cell_check_sum)
+    write_tag(TAG_CELL_CHECKSUM)
+    @output_stream.write_raw_byte(cell_check_sum)
+    row_check_sum = PlainBufferCrc8.crc_int8(row_check_sum, cell_check_sum)
+    row_check_sum
+  end
+
+  def write_column(column_name, column_value, timestamp, row_check_sum)
+    cell_check_sum = 0
+    write_tag(TAG_CELL)
+    cell_check_sum = write_cell_name(column_name, cell_check_sum)
+    cell_check_sum = write_column_value_with_checksum(column_value, cell_check_sum)
+
+    unless timestamp.nil?
+      write_tag(TAG_CELL_TIMESTAMP)
+      @output_stream.write_raw_little_endian64(timestamp)
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, timestamp)
+    end
+    write_tag(TAG_CELL_CHECKSUM)
+    @output_stream.write_raw_byte(cell_check_sum)
+    PlainBufferCrc8.crc_int8(row_check_sum, cell_check_sum)
+  end
+
+  def write_update_column(update_type, column_name, column_value, row_check_sum)
+    update_type = update_type.upcase
+    cell_check_sum = 0
+    write_tag(TAG_CELL)
+    cell_check_sum = write_cell_name(column_name, cell_check_sum)
+    timestamp = nil
+    if column_value.present?
+      if column_value.is_a?(Array)
+        if column_value[0].present?
+          cell_check_sum = write_column_value_with_checksum(column_value[0], cell_check_sum)
+        end
+        if column_value[1].present?
+          timestamp = column_value[1]
+        end
+      else
+        cell_check_sum = write_column_value_with_checksum(column_value, cell_check_sum)
+      end
+    end
+    if update_type == UpdateType.DELETE
+      write_tag(TAG_CELL_TYPE)
+      @output_stream.write_raw_byte(const.DELETE_ONE_VERSION)
+    elsif update_type == UpdateType.DELETE_ALL
+      write_tag(TAG_CELL_TYPE)
+      @output_stream.write_raw_byte(const.DELETE_ALL_VERSION)
+    end
+    if timestamp.present?
+      write_tag(TAG_CELL_TIMESTAMP)
+      @output_stream.write_raw_little_endian64(timestamp)
+    end
+    if timestamp.present?
+      cell_check_sum = PlainBufferCrc8.crc_int64(cell_check_sum, timestamp)
+    end
+    if update_type == UpdateType.DELETE
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, const.DELETE_ONE_VERSION)
+    end
+    if update_type == UpdateType.DELETE_ALL
+      cell_check_sum = PlainBufferCrc8.crc_int8(cell_check_sum, const.DELETE_ALL_VERSION)
+    end
+    write_tag(TAG_CELL_CHECKSUM)
+    @output_stream.write_raw_byte(cell_check_sum)
+    PlainBufferCrc8.crc_int8(row_check_sum, cell_check_sum)
+  end
+
+  def write_primary_key(primary_key, row_check_sum)
+    write_tag(TAG_ROW_PK)
+    primary_key.each do |pk|
+      row_check_sum = write_primary_key_column(pk[0], pk[1], row_check_sum)
+    end
+    row_check_sum
+  end
+
+  def write_columns(columns, row_check_sum)
+    if columns.present? and columns.length != 0
+      write_tag(TAG_ROW_DATA)
+      columns.each do |column|
+        if column.length == 2
+          row_check_sum = write_column(column[0], column[1], nil, row_check_sum)
+        elsif column.length == 3
+          row_check_sum = write_column(column[0], column[1], column[2], row_check_sum)
+        end
+      end
+      row_check_sum
+    end
+  end
+
+  def write_update_columns( attribute_columns, row_check_sum)
+    if attribute_columns.length != 0
+      write_tag(TAG_ROW_DATA)
+      attribute_columns.keys.each do |update_type|
+        columns = attribute_columns[update_type]
+        columns.each do |column|
+          if column.is_a?(String)
+            row_check_sum = write_update_column(update_type, column, nil, row_check_sum)
+          elsif column.length == 2
+            row_check_sum = write_update_column(update_type, column[0], [column[1], nil], row_check_sum)
+          elsif column.length == 3
+            row_check_sum = write_update_column(update_type, column[0], [column[1], column[2]], row_check_sum)
+          else
+              raise TableStoreClientError.new("Unsupported column format: " + column.to_s)
+          end
+        end
+      end
+    end
+    row_check_sum
+  end
+
+  def write_delete_marker(row_checksum)
+    write_tag(TAG_DELETE_ROW_MARKER)
+    PlainBufferCrc8.crc_int8(row_checksum, 1)
+  end
+
+  def write_row_checksum(row_checksum)
+    write_tag(TAG_ROW_CHECKSUM)
+    @output_stream.write_raw_byte(row_checksum)
+  end
+
+  def crc_int8(row_checksum, data)
+    PlainBufferCrc8.crc_int8(row_checksum, data)
+  end
+end