shameless 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3243fd35d7a9377bdaccb46cfcbacd03be44c8c7
-  data.tar.gz: 9d18df882c3964ca57a387199f191dbfdcbbfd61
+  metadata.gz: ffc551d90e92b4f9174e15573bfdefb8a327c20f
+  data.tar.gz: 4e721f69cad576d714069eb2cdb281c13e98d256
 SHA512:
-  metadata.gz: 7b56807b2507d10c86dbdf502b2031fab3beb79024f385e130bca212a15eb6dca6cd2dc0d7a49e2451f947a9875fb8c3b3e423ef894e7429707991469ffdd524
-  data.tar.gz: 9c94897619fd4978e639ff806dcc88625844f13683adc660b28213a67fa64aade9701f23430312ed9b795afa255542beeae0727ce472a7d4f5f3acec0d791231
+  metadata.gz: 2ea84086ac343ccba0b8a44c5c257dfa02130355ffc5efd71dee633db5df9cc6631b15c8d2e24999bf1f211d663caf694664d9e2c5b561708aedd43d91769e1a
+  data.tar.gz: 9d939c4abd011c615cdda23894dd0e82459304b88a487d57de11006f100c3ac0b791fada345520cd6c8d2227e3352d937f69ceb4f1ab379e498d859cf8c8fa1d

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+### 0.3.0 (2016-11-18)
+
+* Add `Cell#uuid`
+* Add `Cell#id`
+* Add `Model.fetch_latest_cells`
+* Initialize `ref_key` with zero, not one
+* Add `Model#present?` and `Cell#present?`
+* Allow `Cell#save` to be called even without making any changes
+* Add `Model#fetch` and `Cell#fetch`
+* Add `Model#reload` and `Cell#reload`
+* Add `Model#previous` and `Cell#previous`
+* `Model.put` now correctly looks up and updates an existing instance
+* Add `Model#update` and `Cell#update`
+* Expose `Model#base`
+* Add `Configuration#legacy_created_at_is_bigint`
+* Keep a reference to only one model class per table name
+* Make `Store#find_shard` public
+* Make `Store#each_shard` public
+* Name index tables `*_:name_index_*`
+* Add `Configuration#database_extensions`, they're being passed to the Sequel adapter
+* Add `Configuration#connection_options`, they're being passed to the Sequel adapter
+* Don't prefix table names with underscore when store name is `nil`
+* Add `Store#each_partition`
+* Add `Store#disconnect`
+
 ### 0.2.0 (2016-11-14)
 
 * Add `Store#padded_shard` to get the formatted shard number for a shardable value

data/README.md

@@ -44,6 +44,8 @@ The core object of shameless is a `Store`. Here's how you can set one up:
 RateStore = Shameless::Store.new(:rate_store) do |c|
   c.partition_urls = [ENV['RATE_STORE_DATABASE_URL_0'], ENV['RATE_STORE_DATABASE_URL_1']
   c.shards_count = 512 # total number of shards across all partitions
+  c.connection_options = {max_connections: 10} # connection options passed to Sequel.connect
+  c.database_extensions = [:newrelic_instrumentation]
 end
 ```
 
@@ -93,7 +95,7 @@ class Rate
 end
 ```
 
-The default index is called a primary index, the corresponding tables would be called `rate_store_rate_primary_[000000-000511]`. You can add additional indices you'd like to query by:
+The default index is called a primary index, the corresponding tables would be called `rate_store_rate_primary_index_[000000-000511]`. You can add additional indices you'd like to query by:
 
 ```ruby
 class Rate

data/lib/shameless/cell.rb

@@ -5,13 +5,20 @@ module Shameless
     BASE = 'base'
 
     def self.base(model, body)
-      new(model, BASE, body)
+      serialized_body = serialize_body(body)
+      new(model, BASE, body: serialized_body)
     end
 
-    def initialize(model, name, body = nil)
+    def self.serialize_body(body)
+      MessagePack.pack(body)
+    end
+
+    attr_reader :model, :name, :id
+
+    def initialize(model, name, values = nil)
       @model = model
       @name = name
-      @body = body
+      initialize_from_values(values)
     end
 
     def [](key)
@@ -24,32 +31,65 @@ module Shameless
     end
 
     def save
+      load
       @created_at = Time.now
-      @ref_key ||= 0
+      @created_at = (@created_at.to_f * 1000).to_i if @model.class.store.configuration.legacy_created_at_is_bigint
+      @ref_key ||= -1
       @ref_key += 1
       @model.put_cell(cell_values)
     end
 
+    def update(values)
+      values.each do |key, value|
+        self[key] = value
+      end
+
+      save
+    end
+
     def ref_key
-      fetch
+      load
       @ref_key
     end
 
     def created_at
-      fetch
+      load
       @created_at
     end
 
     def body
-      fetch
+      load
       @body
     end
 
+    def previous
+      if ref_key && previous_cell_values = @model.fetch_cell(@name, ref_key - 1)
+        self.class.new(@model, @name, previous_cell_values)
+      end
+    end
+
+    def reload
+      @body = @ref_key = @created_at = nil
+    end
+
+    def fetch(key, default)
+      body.key?(key.to_s) ? self[key] : default
+    end
+
+    def present?
+      load
+      !@ref_key.nil?
+    end
+
+    def uuid
+      @model.uuid
+    end
+
     private
 
     def cell_values
       {
-        uuid: @model.uuid,
+        uuid: uuid,
         column_name: @name,
         ref_key: ref_key,
         created_at: created_at,
@@ -58,7 +98,7 @@ module Shameless
     end
 
     def serialized_body
-      MessagePack.pack(body)
+      self.class.serialize_body(body)
     end
 
     def deserialize_body(body)
@@ -67,12 +107,20 @@ module Shameless
 
     private
 
-    def fetch
+    def load
       if @body.nil?
         values = @model.fetch_cell(@name)
-        @ref_key = values[:ref_key] if values
-        @created_at = values[:created_at] if values
-        @body = values ? deserialize_body(values[:body]) : {}
+        initialize_from_values(values)
+        @body ||= {}
+      end
+    end
+
+    def initialize_from_values(values)
+      if values
+        @id = values[:id]
+        @body = deserialize_body(values[:body])
+        @ref_key = values[:ref_key]
+        @created_at = values[:created_at]
       end
     end
   end

data/lib/shameless/configuration.rb

@@ -1,6 +1,8 @@
 module Shameless
   class Configuration
-    attr_accessor :partition_urls, :shards_count
+    attr_accessor :partition_urls, :shards_count, :connection_options, :database_extensions
+
+    attr_accessor :legacy_created_at_is_bigint
 
     def shards_per_partition_count
       shards_count / partitions_count

data/lib/shameless/index.rb

@@ -31,18 +31,33 @@ module Shameless
 
     def put(values)
       shardable_value = values.fetch(@shard_on)
-      index_values = (@columns.keys + [:uuid]).each_with_object({}) {|column, o| o[column] = values.fetch(column) }
+      index_values = index_values(values, true)
 
       @model.store.put(table_name, shardable_value, index_values)
     end
 
     def where(query)
       shardable_value = query.fetch(@shard_on)
+      query = index_values(query, false)
       @model.store.where(table_name, shardable_value, query).map {|r| @model.new(r[:uuid]) }
     end
 
     def table_name
-      "#{@model.table_name}_#{@name}"
+      "#{@model.table_name}_#{full_name}"
+    end
+
+    def full_name
+      "#{@name}_index"
+    end
+
+    def index_values(values, all_required)
+      (@columns.keys + [:uuid]).each_with_object({}) do |column, o|
+        if all_required
+          o[column] = values.fetch(column)
+        else
+          o[column] = values[column] if values.key?(column)
+        end
+      end
     end
 
     def create_tables!
@@ -57,8 +72,12 @@ module Shameless
       end
     end
 
+    def column?(key)
+      @columns.keys.any? {|c| c.to_s == key.to_s }
+    end
+
     def prevent_readonly_attribute_mutation!(key)
-      if @columns.keys.any? {|c| c.to_s == key.to_s }
+      if column?(key)
         raise ReadonlyAttributeMutation, "The attribute #{key} cannot be modified because it's part of the #{@name} index"
       end
     end

data/lib/shameless/model.rb

@@ -17,7 +17,7 @@ module Shameless
       index = Index.new(name, self, &block)
       @indices << index
 
-      define_singleton_method("#{index.name}_index") { index }
+      define_singleton_method(index.full_name) { index }
     end
 
     def cell(name)
@@ -30,13 +30,19 @@ module Shameless
     end
 
     def put(values)
-      uuid = SecureRandom.uuid
-
-      new(uuid, values).tap do |model|
-        model.save
-
-        index_values = values.merge(uuid: uuid)
-        @indices.each {|i| i.put(index_values) }
+      if model = where(values).first
+        model_values = reject_index_values(values)
+        model.update(model_values)
+        model
+      else
+        uuid = SecureRandom.uuid
+
+        new(uuid, values).tap do |model|
+          model.save
+
+          index_values = values.merge(uuid: uuid)
+          @indices.each {|i| i.put(index_values) }
+        end
       end
     end
 
@@ -44,14 +50,28 @@ module Shameless
       @store.put(table_name, shardable_value, cell_values)
     end
 
-    def fetch_cell(shardable_value, uuid, cell_name)
+    def fetch_cell(shardable_value, uuid, cell_name, ref_key)
       query = {uuid: uuid, column_name: cell_name}
+      query[:ref_key] = ref_key if ref_key
 
       @store.where(table_name, shardable_value, query).order(:ref_key).last
     end
 
+    def fetch_latest_cells(shard:, cursor:, limit:)
+      query = ['id > ?', cursor]
+      @store.where(table_name, shard, query).limit(limit).map do |cell_values|
+        model = new(cell_values[:uuid])
+        name = cell_values[:column_name].to_sym
+        Cell.new(model, name, cell_values)
+      end
+    end
+
     def table_name
-      "#{@store.name}_#{@name}"
+      [@store.name, @name].compact.join('_')
+    end
+
+    def table_names
+      [table_name, *@indices.map(&:table_name)]
     end
 
     def create_tables!
@@ -61,7 +81,9 @@ module Shameless
         t.varchar :column_name, null: false
         t.integer :ref_key, null: false
         t.mediumblob :body
-        t.datetime :created_at, null: false
+
+        created_at_type = @store.configuration.legacy_created_at_is_bigint ? :bigint : :datetime
+        t.column :created_at, created_at_type, null: false
 
         t.index %i[uuid column_name ref_key], unique: true
       end
@@ -73,6 +95,10 @@ module Shameless
       primary_index.where(query)
     end
 
+    def reject_index_values(values)
+      values.reject {|k, _| @indices.any? {|i| i.column?(k) } }
+    end
+
     def prevent_readonly_attribute_mutation!(key)
       @indices.each {|i| i.prevent_readonly_attribute_mutation!(key) }
     end
@@ -80,7 +106,7 @@ module Shameless
     private
 
     module InstanceMethods
-      attr_reader :uuid
+      attr_reader :uuid, :base
 
       def initialize(uuid, base_body = nil)
         @uuid = uuid
@@ -95,6 +121,10 @@ module Shameless
         @base[field] = value
       end
 
+      def update(values)
+        @base.update(values)
+      end
+
       def save
         @base.save
       end
@@ -107,12 +137,28 @@ module Shameless
         @base.created_at
       end
 
+      def previous
+        @base.previous
+      end
+
+      def reload
+        @base.reload
+      end
+
+      def fetch(key, default)
+        @base.fetch(key, default)
+      end
+
+      def present?
+        @base.present?
+      end
+
       def put_cell(cell_values)
         self.class.put_cell(shardable_value, cell_values)
       end
 
-      def fetch_cell(cell_name)
-       self.class.fetch_cell(shardable_value, uuid, cell_name)
+      def fetch_cell(cell_name, ref_key = nil)
+       self.class.fetch_cell(shardable_value, uuid, cell_name, ref_key)
       end
 
       def prevent_readonly_attribute_mutation!(key)

data/lib/shameless/store.rb

@@ -4,7 +4,7 @@ require 'shameless/model'
 
 module Shameless
   class Store
-    attr_reader :name
+    attr_reader :name, :configuration
 
     def initialize(name, &block)
       @name = name
@@ -15,8 +15,7 @@ module Shameless
     def attach(model_class, name = nil)
       model_class.extend(Model)
       model_class.attach_to(self, name)
-      @models ||= []
-      @models << model_class
+      models_hash[name] = model_class
     end
 
     def put(table_name, shardable_value, values)
@@ -27,8 +26,20 @@ module Shameless
       find_table(table_name, shardable_value).where(query)
     end
 
+    def disconnect
+      if instance_variable_defined?(:@partitions)
+        partitions.each(&:disconnect)
+      end
+    end
+
+    def each_partition(&block)
+      partitions.each do |partition|
+        block.call(partition, table_names_on_partition(partition))
+      end
+    end
+
     def create_tables!
-      @models.each(&:create_tables!)
+      models.each(&:create_tables!)
     end
 
     def create_table!(table_name, &block)
@@ -44,14 +55,42 @@ module Shameless
       format_shard(shard)
     end
 
+    def each_shard(&block)
+      0.upto(@configuration.shards_count - 1, &block)
+    end
+
+    def find_shard(shardable_value)
+      shardable_value % @configuration.shards_count
+    end
+
     private
 
+    def models_hash
+      @models_hash ||= {}
+    end
+
+    def models
+      models_hash.values
+    end
+
     def partitions
-      @partitions ||= @configuration.partition_urls.map {|url| Sequel.connect(url) }
+      @partitions ||= @configuration.partition_urls.map {|url| connect(url) }
     end
 
-    def each_shard(&block)
-      0.upto(@configuration.shards_count - 1, &block)
+    def connect(url)
+      Sequel.connect(url, @configuration.connection_options || Sequel::OPTS).tap do |db|
+        db.extension *@configuration.database_extensions
+      end
+    end
+
+    def table_names_on_partition(partition)
+      partition_index = partitions.index(partition)
+      first_shard = partition_index * @configuration.shards_per_partition_count
+      last_shard = first_shard + @configuration.shards_per_partition_count - 1
+      shards = first_shard..last_shard
+      table_names = models.flat_map(&:table_names)
+
+      table_names.flat_map {|t| shards.map {|s| table_name_with_shard(t, s) } }
     end
 
     def table_name_with_shard(table_name, shard)
@@ -63,10 +102,6 @@ module Shameless
       shard.to_s.rjust(6, '0')
     end
 
-    def find_shard(shardable_value)
-      shardable_value % @configuration.shards_count
-    end
-
     def find_table(table_name, shardable_value)
       shard = find_shard(shardable_value)
       partition = find_partition_for_shard(shard)

data/lib/shameless/version.rb

@@ -1,3 +1,3 @@
 module Shameless
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shameless
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Olek Janiszewski
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2016-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack