github-ds 0.2.10 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 58267b1fe40225f667e6639dd0df6830b9397853
-  data.tar.gz: 641de968bea62624f96c2dce9233d01e970050b5
+SHA256:
+  metadata.gz: 4b95473248771ff3545335efd43524ac0cabeb2f2ec8ec838f7f7537149e9c5e
+  data.tar.gz: dbf07fa010c19fde19540f22f54e0f6f5fec6d4861b31cac059f92c081f7d9bd
 SHA512:
-  metadata.gz: cd9e18016d02099599946110ae6814e78fc12d82de1d1bf964ad4a9e1fe60d7dd213a20f1867d8c6fe563cdd3731ab6a940db89dbcd8bbe2ac2548fa435e8839
-  data.tar.gz: 6768c6a9631f0f7520370bd879da57d23d3d1c830e0d07349065f12b88896f5e8671aca9be43d1d8cb26a5175cf8394481499f789f6725e35ef8a5a7e6ee3e88
+  metadata.gz: 433f178728cbc1c7748e3b0b7985781bb8d39aabb338169cbc08eb6a954a914a80796b6bcbeda711d1a1de8c0bb87809102b7022c07e2ba97f381bafcdf0c450
+  data.tar.gz: 47586ed750c3af0cbbf0dc34820c99da0c9c69f81946a4f505ba9100c9368ab16e094e0483c7cf37692ad4c0db317b1c0b499c34c5e536e9fb62a1f3e1e4fd1f

data/.travis.yml

@@ -3,13 +3,12 @@ before_install:
   - gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true
   - gem install bundler -v '<2'
 rvm:
-  - 2.3
-  - 2.4
   - 2.5
+  - 2.6
+  - 2.7
 script: bundle exec rake
 env:
-  - RAILS_VERSION=6.0.0.rc1
+  - RAILS_VERSION=6.0.3.5
   - RAILS_VERSION=5.2.0
-  - RAILS_VERSION=5.1.5
-  - RAILS_VERSION=5.0.6
-  - RAILS_VERSION=4.2.10
+services:
+  - mysql

data/Gemfile

@@ -1,12 +1,18 @@
 source "https://rubygems.org"
 gemspec
 
-DEFAULT_RAILS_VERSION = '5.0.2'
+DEFAULT_RAILS_VERSION = '6.0.3.5'
+ENV['RAILS_VERSION'] ||= DEFAULT_RAILS_VERSION
 
-if ENV['RAILS_VERSION'] = '4.2.10'
+if ENV['RAILS_VERSION'] == '4.2.10'
   gem 'mysql2', '~> 0.3.18'
 else
   gem "mysql2"
 end
-gem "rails", "~> #{ENV['RAILS_VERSION'] || DEFAULT_RAILS_VERSION}"
-gem "activerecord", "~> #{ENV['RAILS_VERSION'] || DEFAULT_RAILS_VERSION}"
+
+if ENV['RAILS_VERSION'] == 'master'
+  gem "activerecord", git: "https://github.com/rails/rails"
+else
+  gem "rails", "~> #{ENV['RAILS_VERSION'] || DEFAULT_RAILS_VERSION}"
+  gem "activerecord", "~> #{ENV['RAILS_VERSION'] || DEFAULT_RAILS_VERSION}"
+end

data/README.md

@@ -37,6 +37,14 @@ rails generate github:ds:active_record
 rails db:migrate
 ```
 
+If you need to change the name of the table used for storing the key-values, you can configure your table name as such, before running the migration:
+
+```
+GitHub::KV.configure do |config|
+  config.table_name = "new_key_values_table"
+end
+```
+
 Once you have created and executed the migration, KV can do neat things like this:
 
 ```ruby
@@ -86,6 +94,8 @@ pp kv.mdel(["foo", "bar"])
 # nil
 ```
 
+Note that due to MySQL's default collation, KV keys are case-insensitive.
+
 ### GitHub::SQL
 
 ```ruby

data/github-ds.gemspec

@@ -33,10 +33,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activerecord", ">= 3.2"
 
   spec.add_development_dependency "bundler", ">= 1.14"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "timecop", "~> 0.8.1"
   spec.add_development_dependency "activesupport"
   spec.add_development_dependency "mysql2"
   spec.add_development_dependency "mocha", "~> 1.2.1"
+  spec.add_development_dependency "minitest-focus", "~> 1.1.2"
+  spec.add_development_dependency "pry", "~> 0.12.2"
 end

data/lib/generators/github/ds/active_record_generator.rb

@@ -18,7 +18,7 @@ module Github
         end
 
         def self.migration_version
-          if Rails.version.start_with?('5')
+          if Rails::VERSION::MAJOR.to_i >= 5
             "[#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}]"
           end
         end
@@ -26,6 +26,14 @@ module Github
         def migration_version
           self.class.migration_version
         end
+
+        def self.table_name
+          ":#{GitHub::KV.config.table_name}"
+        end
+
+        def table_name
+          self.class.table_name
+        end
       end
     end
   end

data/lib/generators/github/ds/templates/migration.rb

@@ -1,19 +1,19 @@
 class CreateKeyValuesTable < ActiveRecord::Migration<%= migration_version %>
   def self.up
-    create_table :key_values do |t|
+    create_table <%= table_name %> do |t|
       t.string :key, :null => false
       t.binary :value, :null => false
       t.datetime :expires_at, :null => true
       t.timestamps :null => false
     end
 
-    add_index :key_values, :key, :unique => true
-    add_index :key_values, :expires_at
+    add_index  <%= table_name %>, :key, :unique => true
+    add_index  <%= table_name %>, :expires_at
 
-    change_column :key_values, :id, "bigint(20) NOT NULL AUTO_INCREMENT"
+    change_column  <%= table_name %>, :id, "bigint(20) NOT NULL AUTO_INCREMENT"
   end
 
   def self.down
-    drop_table :key_values
+    drop_table  <%= table_name %>
   end
 end

data/lib/github/ds/version.rb

@@ -1,5 +1,5 @@
 module GitHub
   module DS
-    VERSION = "0.2.10"
+    VERSION = "0.5.2"
   end
 end

data/lib/github/kv.rb

@@ -1,3 +1,4 @@
+require_relative "kv/config"
 require_relative "result"
 require_relative "sql"
 
@@ -48,11 +49,42 @@ module GitHub
     KeyLengthError = Class.new(StandardError)
     ValueLengthError = Class.new(StandardError)
     UnavailableError = Class.new(StandardError)
+    InvalidValueError = Class.new(StandardError)
 
     class MissingConnectionError < StandardError; end
 
-    def initialize(encapsulated_errors = [SystemCallError], &conn_block)
-      @encapsulated_errors = encapsulated_errors
+    attr_accessor :use_local_time
+    attr_writer :config
+
+    def self.config
+      @config ||= Config.new
+    end
+
+    def self.reset
+      @config = Config.new
+    end
+
+    def self.configure
+      yield(config)
+    end
+
+    # initialize :: [Exception], Boolean, Proc -> nil
+    #
+    # Initialize a new KV instance.
+    #
+    # encapsulated_errors - An Array of Exception subclasses that, when raised,
+    #                       will be replaced with UnavailableError.
+    # use_local_time:     - Whether to use Ruby's `Time.now` instaed of MySQL's
+    #                       `NOW()` function. This is mostly useful in testing
+    #                       where time needs to be modified (eg. Timecop).
+    #                       Default false.
+    # &conn_block         - A block to call to open a new database connection.
+    #
+    # Returns nothing.
+    def initialize(config: GitHub::KV.config, &conn_block)
+      @encapsulated_errors = config.encapsulated_errors
+      @use_local_time = config.use_local_time
+      @table_name = config.table_name
       @conn_block = conn_block
     end
 
@@ -93,11 +125,12 @@ module GitHub
       validate_key_array(keys)
 
       Result.new {
-        kvs = GitHub::SQL.results(<<-SQL, :keys => keys, :connection => connection).to_h
-          SELECT `key`, value FROM key_values WHERE `key` IN :keys AND (`expires_at` IS NULL OR `expires_at` > NOW())
+        kvs = GitHub::SQL.results(<<-SQL, :keys => keys, :now => now, :connection => connection).to_h
+          SELECT `key`, value FROM #{@table_name} WHERE `key` IN :keys AND (`expires_at` IS NULL OR `expires_at` > :now)
         SQL
 
-        keys.map { |key| kvs[key] }
+        kvs.keys.each { |key| kvs[key.downcase] = kvs[key] }
+        keys.map { |key| kvs[key.downcase] }
       }
     end
 
@@ -137,12 +170,12 @@ module GitHub
 
       rows = kvs.map { |key, value|
         value = value.is_a?(GitHub::SQL::Literal) ? value : GitHub::SQL::BINARY(value)
-        [key, value, GitHub::SQL::NOW, GitHub::SQL::NOW, expires || GitHub::SQL::NULL]
+        [key, value, now, now, expires || GitHub::SQL::NULL]
       }
 
       encapsulate_error do
         GitHub::SQL.run(<<-SQL, :rows => GitHub::SQL::ROWS(rows), :connection => connection)
-          INSERT INTO key_values (`key`, value, created_at, updated_at, expires_at)
+          INSERT INTO #{@table_name} (`key`, value, created_at, updated_at, expires_at)
           VALUES :rows
           ON DUPLICATE KEY UPDATE
             value = VALUES(value),
@@ -186,8 +219,8 @@ module GitHub
       validate_key_array(keys)
 
       Result.new {
-        existing_keys = GitHub::SQL.values(<<-SQL, :keys => keys, :connection => connection).to_set
-          SELECT `key` FROM key_values WHERE `key` IN :keys AND (`expires_at` IS NULL OR `expires_at` > NOW())
+        existing_keys = GitHub::SQL.values(<<-SQL, :keys => keys, :now => now, :connection => connection).to_set
+          SELECT `key` FROM #{@table_name} WHERE `key` IN :keys AND (`expires_at` IS NULL OR `expires_at` > :now)
         SQL
 
         keys.map { |key| existing_keys.include?(key) }
@@ -222,20 +255,116 @@ module GitHub
         # achieve the same thing with the right INSERT ... ON DUPLICATE KEY UPDATE
         # query, but then we would not be able to rely on affected_rows
 
-        GitHub::SQL.run(<<-SQL, :key => key, :connection => connection)
-          DELETE FROM key_values WHERE `key` = :key AND expires_at <= NOW()
+        GitHub::SQL.run(<<-SQL, :key => key, :now => now, :connection => connection)
+          DELETE FROM #{@table_name} WHERE `key` = :key AND expires_at <= :now
         SQL
 
         value = value.is_a?(GitHub::SQL::Literal) ? value : GitHub::SQL::BINARY(value)
-        sql = GitHub::SQL.run(<<-SQL, :key => key, :value => value, :expires => expires || GitHub::SQL::NULL, :connection => connection)
-          INSERT IGNORE INTO key_values (`key`, value, created_at, updated_at, expires_at)
-          VALUES (:key, :value, NOW(), NOW(), :expires)
+        sql = GitHub::SQL.run(<<-SQL, :key => key, :value => value, :now => now, :expires => expires || GitHub::SQL::NULL, :connection => connection)
+          INSERT IGNORE INTO #{@table_name} (`key`, value, created_at, updated_at, expires_at)
+          VALUES (:key, :value, :now, :now, :expires)
         SQL
 
         sql.affected_rows > 0
       }
     end
 
+    # increment :: String, Integer, expires: Time? -> Integer
+    #
+    # Increment the key's value by an amount.
+    #
+    # key             - The key to increment.
+    # amount          - The amount to increment the key's value by.
+    #                   The user can increment by both positive and
+    #                   negative values
+    # expires         - When the key should expire.
+    # touch_on_insert - Only when expires is specified. When true
+    #                   the expires value is only touched upon
+    #                   inserts. Otherwise the record is always
+    #                   touched.
+    #
+    # Returns the key's value after incrementing.
+    def increment(key, amount: 1, expires: nil, touch_on_insert: false)
+      validate_key(key)
+      validate_amount(amount) if amount
+      validate_expires(expires) if expires
+      validate_touch(touch_on_insert, expires)
+
+      expires ||= GitHub::SQL::NULL
+
+      # This query uses a few MySQL "hacks" to ensure that the incrementing
+      # is done atomically and the value is returned. The first trick is done
+      # using the `LAST_INSERT_ID` function. This allows us to manually set
+      # the LAST_INSERT_ID returned by the query. Here we are able to set it
+      # to the new value when an increment takes place, essentially allowing us
+      # to do: `UPDATE...;SELECT value from key_value where key=:key` in a
+      # single step.
+      #
+      # However the `LAST_INSERT_ID` trick is only used when the value is
+      # updated. Upon a fresh insert we know the amount is going to be set
+      # to the amount specified.
+      #
+      # Lastly we only do these tricks when the value at the key is an integer.
+      # If the value is not an integer the update ensures the values remain the
+      # same and we raise an error.
+      encapsulate_error {
+        sql = GitHub::SQL.run(<<-SQL, key: key, amount: amount, now: now, expires: expires, touch: !touch_on_insert, connection: connection)
+          INSERT INTO #{@table_name} (`key`, `value`, `created_at`, `updated_at`, `expires_at`)
+          VALUES(:key, :amount, :now, :now, :expires)
+          ON DUPLICATE KEY UPDATE
+            `value`=IF(
+              concat('',`value`*1) = `value`,
+              LAST_INSERT_ID(IF(
+                `expires_at` IS NULL OR `expires_at`>=:now,
+                `value`+:amount,
+                :amount
+              )),
+              `value`
+            ),
+            `updated_at`=IF(
+              concat('',`value`*1) = `value`,
+              :now,
+              `updated_at`
+            ),
+            `expires_at`=IF(
+              concat('',`value`*1) = `value`,
+              IF(
+                :touch OR (`expires_at` IS NULL OR `expires_at`<:now),
+                :expires,
+                `expires_at`
+              ),
+              `expires_at`
+            )
+        SQL
+
+        # The ordering of these statements is extremely important if we are to
+        # support incrementing a negative amount. The checks occur in this order:
+        # 1. Check if an update with new values occurred? If so return the result
+        #    This could potentially result in `sql.last_insert_id` with a value
+        #    of 0, thus it must be before the second check.
+        # 2. Check if an update took place but nothing changed (I.E. no new value
+        #    was set)
+        # 3. Check if an insert took place.
+        #
+        # See https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html for
+        # more information (NOTE: CLIENT_FOUND_ROWS is set)
+        if sql.affected_rows == 2
+          # An update took place in which data changed. We use a hack to set
+          # the last insert ID to be the new value.
+          sql.last_insert_id
+        elsif sql.affected_rows == 0 || (sql.affected_rows == 1 && sql.last_insert_id == 0)
+          # No insert took place nor did any update occur. This means that
+          # the value was not an integer thus not incremented.
+          raise InvalidValueError
+        elsif sql.affected_rows == 1
+          # If the number of affected_rows is 1 then a new value was inserted
+          # thus we can just return the amount given to us since that is the
+          # value at the key
+          amount
+        end
+      }
+    end
+
     # del :: String -> nil
     #
     # Deletes the specified key. Returns nil. Raises on error.
@@ -265,7 +394,7 @@ module GitHub
 
       encapsulate_error do
         GitHub::SQL.run(<<-SQL, :keys => keys, :connection => connection)
-          DELETE FROM key_values WHERE `key` IN :keys
+          DELETE FROM #{@table_name} WHERE `key` IN :keys
         SQL
       end
 
@@ -288,14 +417,40 @@ module GitHub
       validate_key(key)
 
       Result.new {
-        GitHub::SQL.value(<<-SQL, :key => key, :connection => connection)
-          SELECT expires_at FROM key_values
-          WHERE `key` = :key AND (expires_at IS NULL OR expires_at > NOW())
+        GitHub::SQL.value(<<-SQL, :key => key, :now => now, :connection => connection)
+          SELECT expires_at FROM #{@table_name}
+          WHERE `key` = :key AND (expires_at IS NULL OR expires_at > :now)
         SQL
       }
     end
 
+    # mttl :: [String] -> Result<[Time | nil]>
+    #
+    # Returns the expires_at time for the specified key or nil.
+    #
+    # Example:
+    #
+    #  kv.mttl(["foo", "octocat"])
+    #    # => #<Result value: [2018-04-23 11:34:54 +0200, nil]>
+    #
+    def mttl(keys)
+      validate_key_array(keys)
+
+      Result.new {
+        kvs = GitHub::SQL.results(<<-SQL, :keys => keys, :now => now, :connection => connection).to_h
+          SELECT `key`, expires_at FROM #{@table_name}
+          WHERE `key` in :keys AND (expires_at IS NULL OR expires_at > :now)
+        SQL
+
+        keys.map { |key| kvs[key] }
+      }
+    end
+
   private
+    def now
+      use_local_time ? Time.now : GitHub::SQL::NOW
+    end
+
     def validate_key(key, error_message: nil)
       unless key.is_a?(String)
         raise TypeError, error_message || "key must be a String in #{self.class.name}, but was #{key.class}"
@@ -349,6 +504,19 @@ module GitHub
       end
     end
 
+    def validate_amount(amount)
+      raise ArgumentError.new("The amount specified must be an integer") unless amount.is_a? Integer
+      raise ArgumentError.new("The amount specified cannot be 0") if amount == 0
+    end
+
+    def validate_touch(touch, expires)
+      raise ArgumentError.new("touch_on_insert must be a boolean value") unless [true, false].include?(touch)
+
+      if touch && expires.nil?
+        raise ArgumentError.new("Please specify an expires value if you wish to touch on insert")
+      end
+    end
+
     def validate_expires(expires)
       unless expires.respond_to?(:to_time)
         raise TypeError, "expires must be a time of some sort (Time, DateTime, ActiveSupport::TimeWithZone, etc.), but was #{expires.class}"

data/lib/github/kv/config.rb

@@ -0,0 +1,13 @@
+module GitHub
+  class KV
+    class Config
+      attr_accessor :table_name, :encapsulated_errors, :use_local_time
+
+      def initialize
+        @table_name = 'key_values'
+        @encapsulated_errors = [SystemCallError]
+        @use_local_time = false
+      end
+    end
+  end
+end

data/lib/github/sql.rb

@@ -43,7 +43,7 @@ module GitHub
     # GitHub::SQL#initialize or overriding connection then you'll need to use
     # the instance version.
     def self.transaction(options = {}, &block)
-      ActiveRecord::Base.connection.transaction(options, &block)
+      ActiveRecord::Base.connection.transaction(**options, &block)
     end
 
     # Public: Instantiate a literal SQL value.
@@ -213,7 +213,7 @@ module GitHub
       # Use select_all to retrieve hashes for each row instead of arrays of values.
       @models = connection.
         select_all(query, "#{klass.name} Load via #{self.class.name}").
-          collect! { |record| klass.send :instantiate, record }
+        map { |record| klass.send :instantiate, record }
 
       retrieve_found_row_count
       freeze
@@ -299,7 +299,7 @@ module GitHub
 
     # Public: Run inside a transaction for the connection.
     def transaction(options = {}, &block)
-      connection.transaction(options, &block)
+      connection.transaction(**options, &block)
     end
 
     # Internal: The object we use to execute SQL and retrieve results. Defaults

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: github-ds
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.5.2
 platform: ruby
 authors:
 - GitHub Open Source
 - John Nunemaker
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-05-17 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,14 +45,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +123,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.2
 description: A collection of libraries for working with SQL on top of ActiveRecord's
   connection.
 email:
@@ -155,6 +183,7 @@ files:
 - lib/github/ds.rb
 - lib/github/ds/version.rb
 - lib/github/kv.rb
+- lib/github/kv/config.rb
 - lib/github/result.rb
 - lib/github/sql.rb
 - lib/github/sql/errors.rb
@@ -170,7 +199,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -185,9 +214,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: A collection of libraries for working with SQL on top of ActiveRecord's connection.
 test_files: []