ok_hbase 0.0.5

data/lib/ok_hbase/concerns/table/batch.rb

@@ -0,0 +1,95 @@
+module OkHbase
+  module Concerns
+    module Table
+      class Batch
+
+        attr_accessor :batch_wrapper
+        attr_reader :table, :batch_size, :timestamp
+
+        def initialize(table, timestamp = nil, batch_size = nil, transaction = false, &batch_wrapper)
+          raise TypeError.new "'timestamp' must be an integer or nil" if timestamp && !timestamp.is_a?(Integer)
+
+          if batch_size
+            raise ArgumentError.new "'transaction' cannot be used when 'batch_size' is specified" if transaction
+            raise ValueError.new "'batch_size' must be > 0" unless batch_size > 0
+          end
+
+          @table = table
+          @batch_size = batch_size
+          @timestamp = timestamp
+          @transaction = transaction
+          @batch_wrapper = batch_wrapper
+          @families = nil
+
+          _reset_mutations()
+
+        end
+
+        def send_batch()
+          if batch_wrapper
+            batch_wrapper.call(@mutations) do
+              _send_batch
+            end
+          else
+            _send_batch
+          end
+        end
+
+        def put(row_key, data)
+          @mutations[row_key] ||= []
+
+          data.each_pair do |column, value|
+            @mutations[row_key] << Apache::Hadoop::Hbase::Thrift::Mutation.new(
+                isDelete: false, column: column, value: value
+            )
+          end
+
+          @mutation_count += data.size
+
+          send_batch if @batch_size && @mutation_count > @batch_size
+        end
+
+        def delete(row_key, columns = nil)
+          columns ||= @families ||= @table.send(:_column_family_names)
+
+          @mutations[row_key] ||= []
+
+          columns.each do |column|
+            @mutations[row_key] << Apache::Hadoop::Hbase::Thrift::Mutation.new(isDelete: true, column: column)
+          end
+
+          @mutation_count += columns.size
+          send_batch if @batch_size && @mutation_count > @batch_size
+        end
+
+        def transaction
+          yield self
+          send_batch
+        end
+
+        private
+
+        def _reset_mutations
+          @mutations = {}
+          @mutation_count = 0
+        end
+
+        def _send_batch()
+          batch_mutations = @mutations.map do |row_key, mutations|
+            Apache::Hadoop::Hbase::Thrift::BatchMutation.new(row: row_key.dup.force_encoding(Encoding::UTF_8), mutations: mutations)
+          end
+
+          return if batch_mutations.blank?
+
+          if @timestamp
+            @table.connection.client.mutateRowsTs(@table.connection.table_name(@table.table_name), batch_mutations, @timestamp)
+          else
+            @table.connection.client.mutateRows(@table.connection.table_name(@table.table_name), batch_mutations)
+          end
+
+          _reset_mutations()
+        end
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/table/class_methods.rb

@@ -0,0 +1,13 @@
+module OkHbase
+  module Concerns
+    module Table
+      module ClassMethods
+        extend ActiveSupport::Concern
+
+        module ClassMethods
+          include OkHbase::Concerns::Table
+        end
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/table/instrumentation.rb

@@ -0,0 +1,48 @@
+module OkHbase
+  module Concerns
+    module Table
+      module Instrumentation
+        extend ActiveSupport::Concern
+
+        module ClassMethods
+          instrumented_methods = {
+              load: %w[row rows cells scan],
+          }
+
+          instrumented_methods.each_pair do |method_type, methods|
+            methods.each do |method_to_instrument|
+              module_eval <<-RUBY, __FILE__, __LINE__
+              def #{method_to_instrument}(*args)
+                opts = {
+                  name: [table_name, '#{method_type}'].join(' '),
+                  description: "#{method_to_instrument}",
+                  options: args
+                }
+                ActiveSupport::Notifications.instrument("#{method_type}.ok_hbase", opts) do
+                  super
+                end
+              end
+              RUBY
+            end
+          end
+
+          def batch(timestamp = nil, batch_size = nil, transaction = false)
+
+             batch_wrapper = Proc.new do |*args, &block|
+              opts = {
+                  name: [table_name, 'write'].join(' '),
+                  description: "send_batch",
+                  options: args
+              }
+              ActiveSupport::Notifications.instrument("write.ok_hbase", opts, &block)
+            end
+
+            batch = Batch.new(self, timestamp, batch_size, transaction)
+            batch.batch_wrapper = batch_wrapper
+            batch
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/table.rb

@@ -0,0 +1,241 @@
+require 'ok_hbase/concerns/table/batch'
+
+module OkHbase
+  module Concerns
+    module Table
+      extend ActiveSupport::Concern
+
+      SCANNER_DEFAULTS = {
+          start_row: nil,
+          stop_row: nil,
+          row_prefix: nil,
+          columns: nil,
+          filter_string: nil,
+          timestamp: nil,
+          include_timestamp: false,
+          caching: 1000,
+          limit: nil,
+      }.freeze
+
+      attr_accessor :table_name, :connection
+
+      def table_name
+        @table_name
+      end
+
+      def table_name=(val)
+        @table_name = val
+      end
+
+      def self.connection
+        @self.connection
+      end
+
+      def self.connection=(val)
+        @self.connection = val
+      end
+
+
+      def families()
+        descriptors = self.connection.client.getColumnDescriptors(self.connection.table_name(table_name))
+
+        families = {}
+
+        descriptors.each_pair do |name, descriptor|
+          name = name[0...-1] # remove trailing ':'
+          families[name] = OkHbase.thrift_type_to_dict(descriptor)
+        end
+        families
+      end
+
+      def regions
+        regions = self.connection.client.getTableRegions(self.connection.table_name(table_name))
+        regions.map { |r| OkHbase.thrift_type_to_dict(r) }
+      end
+
+      def row(row_key, columns = nil, timestamp = nil, include_timestamp = false)
+        raise TypeError.new "'columns' must be a tuple or list" if columns && !columns.is_a?(Array)
+
+        row_key.force_encoding(Encoding::UTF_8)
+
+        rows = if timestamp
+          raise TypeError.new "'timestamp' must be an integer" unless timestamp.is_a? Integer
+
+          self.connection.client.getRowWithColumnsTs(self.connection.table_name(table_name), row_key, columns, timestamp)
+        else
+          self.connection.client.getRowWithColumns(self.connection.table_name(table_name), row_key, columns)
+        end
+
+        rows.empty? ? {} : _make_row(rows[0].columns, include_timestamp)
+      end
+
+      def rows(row_keys, columns = nil, timestamp = nil, include_timestamp = false)
+        raise TypeError.new "'columns' must be a tuple or list" if columns && !columns.is_a?(Array)
+
+        row_keys.map! { |r| r.force_encoding(Encoding::UTF_8) }
+
+        return {} if row_keys.blank?
+
+        rows = if timestamp
+          raise TypeError.new "'timestamp' must be an integer" unless timestamp.is_a? Integer
+
+          columns = _column_family_names() unless columns
+
+          self.connection.client.getRowsWithColumnsTs(self.connection.table_name(table_name), row_keys, columns, timestamp)
+        else
+          self.connection.client.getRowsWithColumns(self.connection.table_name(table_name), row_keys, columns)
+        end
+
+        rows.map { |row| _make_row(row.columns, include_timestamp) }
+      end
+
+      def cells(row_key, column, versions = nil, timestamp = nil, include_timestamp = nil)
+
+        row_key.force_encoding(Encoding::UTF_8)
+
+        versions ||= (2 ** 31) -1
+
+        raise TypeError.new "'versions' parameter must be a number or nil" unless versions.is_a? Integer
+        raise ArgumentError.new "'versions' parameter must be >= 1" unless versions >= 1
+
+        cells = if timestamp
+          raise TypeError.new "'timestamp' must be an integer" unless timestamp.is_a? Integer
+
+          self.connection.client.getVerTs(self.connection.table_name(table_name), row_key, column, timestamp, versions)
+        else
+          self.connection.client.getVer(self.connection.table_name(table_name), row_key, column, versions)
+        end
+
+        cells.map { |cell| include_timestamp ? [cell.value, cell.timestamp] : cell.value }
+      end
+
+      def scan(opts={})
+
+        rows = [] unless block_given?
+        opts = SCANNER_DEFAULTS.merge opts.select { |k| SCANNER_DEFAULTS.keys.include? k }
+
+
+        raise ArgumentError.new "'caching' must be >= 1" unless opts[:caching] && opts[:caching] >= 1
+        raise ArgumentError.new "'limit' must be >= 1" if opts[:limit] && opts[:limit] < 1
+
+        if opts[:row_prefix]
+          raise ArgumentError.new "'row_prefix' cannot be combined with 'start_row' or 'stop_row'" if opts[:start_row] || opts[:stop_row]
+
+          opts[:start_row] = opts[:row_prefix]
+          opts[:stop_row] = OkHbase::increment_string opts[:start_row]
+
+        end
+        opts[:start_row] ||= ''
+
+        scanner = _scanner(opts)
+
+        scanner_id = self.connection.client.scannerOpenWithScan(self.connection.table_name(table_name), scanner)
+
+        fetched_count = returned_count = 0
+
+        begin
+          while true
+            how_many = opts[:limit] ? [opts[:caching], opts[:limit] - returned_count].min : opts[:caching]
+
+            items = if how_many == 1
+              self.connection.client.scannerGet(scanner_id)
+            else
+              self.connection.client.scannerGetList(scanner_id, how_many)
+            end
+
+            fetched_count += items.length
+
+            items.map.with_index do |item, index|
+              if block_given?
+                yield item.row, _make_row(item.columns, opts[:include_timestamp])
+              else
+                rows << [item.row, _make_row(item.columns, opts[:include_timestamp])]
+              end
+              return rows if opts[:limit] && index + 1 + returned_count == opts[:limit]
+            end
+
+            break if items.length < how_many
+          end
+        ensure
+          self.connection.client.scannerClose(scanner_id)
+        end
+        rows
+      end
+
+      def put(row_key, data, timestamp = nil)
+        batch = self.batch(timestamp)
+
+        batch.transaction do |batch|
+          batch.put(row_key, data)
+        end
+      end
+
+      def delete(row_key, columns = nil, timestamp = nil)
+        if columns
+          batch = self.batch(timestamp)
+          batch.transaction do |batch|
+            batch.delete(row_key, columns)
+          end
+
+        else
+          timestamp ? self.connection.client.deleteAllRowTs(self.connection.table_name(table_name), row_key, timestamp) : self.connection.client.deleteAllRow(self.connection.table_name(table_name), row_key)
+        end
+      end
+
+      def batch(timestamp = nil, batch_size = nil, transaction = false)
+        Batch.new(self, timestamp, batch_size, transaction)
+      end
+
+      def counter_get(row_key, column)
+        counter_inc(row_key, column, 0)
+      end
+
+      def counter_set(row_key, column, value = 0)
+        self.batch.transaction do |batch|
+          batch.put(row_key, { column => [value].pack('Q>') })
+        end
+      end
+
+      def counter_inc(row_key, column, value = 1)
+        self.connection.client.atomicIncrement(self.connection.table_name(table_name), row_key, column, value)
+      end
+
+      def counter_dec(row_key, column, value = 1)
+        counter_inc(row_key, column, -value)
+      end
+
+      alias_method :find, :scan
+
+      def _column_family_names()
+        self.connection.client.getColumnDescriptors(self.connection.table_name(table_name)).keys()
+      end
+
+      def _scanner(opts)
+        scanner = Apache::Hadoop::Hbase::Thrift::TScan.new()
+        scanner_fields = Apache::Hadoop::Hbase::Thrift::TScan::FIELDS
+
+        opts.each_pair do |k, v|
+          const = k.to_s.upcase.gsub('_', '')
+          const_value = Apache::Hadoop::Hbase::Thrift::TScan.const_get(const) rescue nil
+
+          if const_value
+            v.force_encoding(Encoding::UTF_8) if v.is_a?(String)
+            OkHbase.logger.info "setting scanner.#{scanner_fields[const_value][:name]}: #{v}"
+            scanner.send("#{scanner_fields[const_value][:name]}=", v)
+          else
+          end
+        end
+        scanner
+
+      end
+
+      def _make_row(cell_map, include_timestamp)
+        row = {}
+        cell_map.each_pair do |cell_name, cell|
+          row[cell_name] = include_timestamp ? [cell.value, cell.timestamp] : cell.value
+        end
+        row
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns.rb

@@ -0,0 +1,13 @@
+module OkHbase
+  module Concerns
+
+  end
+end
+require 'ok_hbase/concerns/table'
+require 'ok_hbase/concerns/table/class_methods'
+require 'ok_hbase/concerns/table/instrumentation'
+require 'ok_hbase/concerns/row'
+require 'ok_hbase/concerns/custom_row'
+require 'ok_hbase/concerns/custom_row/class_methods'
+require 'ok_hbase/concerns/indexable'
+require 'ok_hbase/concerns/indexable/class_methods'

data/lib/ok_hbase/connection.rb

@@ -0,0 +1,157 @@
+require 'thrift'
+require 'thrift/transport/socket'
+require 'thrift/protocol/binary_protocol'
+
+require 'thrift/hbase/hbase_constants'
+require 'thrift/hbase/hbase_types'
+require 'thrift/hbase/hbase'
+
+require 'ok_hbase/client'
+
+module OkHbase
+  class Connection
+
+
+    DEFAULT_OPTS = {
+        host: 'localhost',
+        port: 9090,
+        timeout: 5,
+        auto_connect: false,
+        table_prefix: nil,
+        table_prefix_separator: '_',
+        transport: :buffered,
+        max_tries: 3
+    }.freeze
+
+    THRIFT_TRANSPORTS = {
+        buffered: Thrift::BufferedTransport,
+        framed: Thrift::FramedTransport,
+    }
+
+    attr_accessor :host, :port, :timeout, :auto_connect, :table_prefix, :table_prefix_separator, :max_tries
+    attr_reader :client
+
+    def initialize(opts={})
+      opts = DEFAULT_OPTS.merge opts
+
+      raise ArgumentError.new ":transport must be one of: #{THRIFT_TRANSPORTS.keys}" unless THRIFT_TRANSPORTS.keys.include?(opts[:transport])
+      raise TypeError.new ":table_prefix must be a string" if opts[:table_prefix] && !opts[:table_prefix].is_a?(String)
+      raise TypeError.new ":table_prefix_separator must be a string" unless opts[:table_prefix_separator].is_a?(String)
+
+
+      @host = opts[:host]
+      @port = opts[:port]
+      @timeout = opts[:timeout]
+      @max_tries = opts[:max_tries]
+      @auto_connect = opts[:auto_connect]
+      @table_prefix = opts[:table_prefix]
+      @table_prefix_separator = opts[:table_prefix_separator]
+      @transport_class = THRIFT_TRANSPORTS[opts[:transport]]
+
+      _refresh_thrift_client
+      open if @auto_connect
+
+    end
+
+    def open
+      return if open?
+      @transport.open
+
+      OkHbase.logger.info "OkHbase connected"
+    end
+
+    def open?
+      @transport && @transport.open?
+    end
+
+    def close
+      return unless open?
+      @transport.close
+    end
+
+    def table(name, use_prefix=true)
+      name = table_name(name) if use_prefix
+      OkHbase::Table.new(name, self)
+    end
+
+    def tables
+      names = client.getTableNames
+      if table_prefix
+        names = names.map do |n|
+          n["#{table_prefix}#{table_prefix_separator}".size..-1] if n.start_with?(table_prefix)
+        end
+      end
+      names
+    end
+
+    def create_table(name, families)
+      name = table_name(name)
+
+      raise ArgumentError.new "Can't create table #{name}. (no column families specified)" unless families
+      raise TypeError.new "families' arg must be a hash" unless families.respond_to?(:[])
+
+      column_descriptors = []
+
+      families.each_pair do |family_name, options|
+        options ||= {}
+
+        args = {}
+        options.each_pair do |option_name, value|
+          args[option_name.to_s.camelcase(:lower)] = value
+        end
+
+        family_name = "#{family_name}:" unless family_name.to_s.end_with? ':'
+        args[:name] = family_name
+
+        column_descriptors << Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new(args)
+      end
+
+      client.createTable(name, column_descriptors)
+      table(name)
+    end
+
+    def delete_table(name, disable=false)
+      name = table_name(name)
+
+      disable_table(name) if disable && table_enabled?(name)
+      client.deleteTable(name)
+    end
+
+    def enable_table(name)
+      name = table_name(name)
+
+      client.enableTable(name)
+    end
+
+    def disable_table(name)
+      name = table_name(name)
+
+      client.disableTable(name)
+    end
+
+    def table_enabled?(name)
+      name = table_name(name)
+
+      client.isTableEnabled(name)
+    end
+
+    def compact_table(name, major=false)
+      name = table_name(name)
+
+      major ? client.majorCompact(name) : client.compact(name)
+    end
+
+    def table_name(name)
+      table_prefix && !name.start_with?(table_prefix) ? [table_prefix, name].join(table_prefix_separator) : name
+    end
+
+    private
+
+    def _refresh_thrift_client
+      socket = Thrift::Socket.new(host, port, timeout)
+      @transport = @transport_class.new(socket)
+      protocol = Thrift::BinaryProtocolAccelerated.new(@transport)
+      @client = OkHbase::Client.new(protocol, nil, max_tries)
+    end
+  end
+end

data/lib/ok_hbase/row.rb

@@ -0,0 +1,21 @@
+module OkHbase
+  class Row
+    include OkHbase::Concerns::Row
+
+    def initialize(opts={})
+      opts = opts.with_indifferent_access
+
+      raise ArgumentError.new "'table' must be an OkHBase::Concerns::Table. is a #{opts[:table].class}" unless opts[:table] && opts[:table].is_a?(OkHbase::Concerns::Table)
+      @default_column_family = opts[:default_column_family]
+
+      @table = opts[:table]
+
+      @row_key = opts[:row_key]
+      @raw_data = {}.with_indifferent_access
+      opts[:raw_data].each_pair do |k, v|
+
+        send(:"#{k}=", v)
+      end
+    end
+  end
+end

data/lib/ok_hbase/table.rb

@@ -0,0 +1,10 @@
+module OkHbase
+  class Table
+    include OkHbase::Concerns::Table
+
+    def initialize(name, connection)
+      @connection = connection
+      @table_name = name
+    end
+  end
+end

data/lib/ok_hbase/version.rb

@@ -0,0 +1,3 @@
+module OkHbase
+    VERSION = "0.0.5"
+end

data/lib/ok_hbase.rb

@@ -0,0 +1,39 @@
+require 'active_support/all'
+
+
+module OkHbase
+
+  mattr_accessor :logger
+
+  def self.root
+    ::Pathname.new File.expand_path('../../', __FILE__)
+  end
+
+  def self.logger
+    @@logger ||= init_logger
+  end
+
+  def self.init_logger
+    Logger.new("/dev/null")
+  end
+
+  def self.increment_string(string)
+    bytes = string.bytes.to_a
+    (0...bytes.length).to_a.reverse.each do |i|
+      return (bytes[0...i] << bytes[i]+1).pack('C*').force_encoding(Encoding::UTF_8) unless bytes[i] == 255
+    end
+    nil
+  end
+
+  def self.thrift_type_to_dict(obj)
+    Hash[obj.class::FIELDS.map{ |k, v| [v[:name].underscore, obj.send(v[:name])]}]
+  end
+end
+
+require 'ok_hbase/version'
+require 'ok_hbase/client'
+require 'ok_hbase/connection'
+require 'ok_hbase/concerns'
+require 'ok_hbase/table'
+require 'ok_hbase/row'
+require 'ok_hbase/active_model'