github-ds 0.1.0

data/examples/result.rb

@@ -0,0 +1,50 @@
+require File.expand_path("../example_setup", __FILE__)
+require "github/result"
+
+def do_something
+  1
+end
+
+def do_something_that_errors
+  raise "noooooppppeeeee"
+end
+
+result = GitHub::Result.new { do_something }
+p result.ok? # => true
+p result.value! # => 1
+
+result = GitHub::Result.new { do_something_that_errors }
+p result.ok? # => false
+p result.value { "default when error happens" } # => "default when error happens"
+
+begin
+  result.value! # => raises exception because error happened
+rescue => error
+  p result.error # => the error
+  p error # the same error
+end
+
+# Outputs Step 1, 2, 3
+result = GitHub::Result.new {
+  GitHub::Result.new { puts "Step 1: success!" }
+}.then { |value|
+  GitHub::Result.new { puts "Step 2: success!" }
+}.then { |value|
+  GitHub::Result.new { puts "Step 3: success!" }
+}
+p result.ok? # => true
+
+# Outputs Step 1, 2 and stops.
+result = GitHub::Result.new {
+  GitHub::Result.new { puts "Step 1: success!" }
+}.then { |value|
+  GitHub::Result.new {
+    puts "Step 2: failed!"
+    raise
+  }
+}.then { |value|
+  GitHub::Result.new {
+    puts "Step 3: should not get here because previous step failed!"
+  }
+}
+p result.ok? # => false

data/examples/sql.rb

@@ -0,0 +1,44 @@
+require File.expand_path("../example_setup", __FILE__)
+require "github/sql"
+
+insert_statement = "INSERT INTO example_key_values (`key`, `value`) VALUES (:key, :value)"
+select_statement = "SELECT value FROM example_key_values WHERE `key` = :key"
+update_statement = "UPDATE example_key_values SET value = :value WHERE `key` = :key"
+delete_statement = "DELETE FROM example_key_values WHERE `key` = :key"
+
+################################# Class Style ##################################
+sql = GitHub::SQL.run insert_statement, key: "foo", value: "bar"
+p sql.last_insert_id
+# 1
+
+p GitHub::SQL.value select_statement, key: "foo"
+# "bar"
+
+sql = GitHub::SQL.run update_statement, key: "foo", value: "new value"
+p sql.affected_rows
+# 1
+
+sql = GitHub::SQL.run delete_statement, key: "foo"
+p sql.affected_rows
+# 1
+
+
+################################ Instance Style ################################
+sql = GitHub::SQL.new insert_statement, key: "foo", value: "bar"
+sql.run
+p sql.last_insert_id
+# 2
+
+sql = GitHub::SQL.new select_statement, key: "foo"
+p sql.value
+# "bar"
+
+sql = GitHub::SQL.new update_statement, key: "foo", value: "new value"
+sql.run
+p sql.affected_rows
+# 1
+
+sql = GitHub::SQL.new delete_statement, key: "foo"
+sql.run
+p sql.affected_rows
+# 1

data/examples/sql_add.rb

@@ -0,0 +1,42 @@
+require File.expand_path("../example_setup", __FILE__)
+require "github/sql"
+
+GitHub::SQL.run <<-SQL
+  INSERT INTO example_key_values (`key`, `value`)
+  VALUES ("foo", "bar"), ("baz", "wick")
+SQL
+
+sql = GitHub::SQL.new "SELECT `VALUE` FROM example_key_values"
+
+key = ENV["KEY"]
+unless key.nil?
+  sql.add "WHERE `key` = :key", key: key
+end
+
+limit = ENV["LIMIT"]
+unless limit.nil?
+  sql.add "ORDER BY `key` ASC"
+  sql.add "LIMIT :limit", limit: limit.to_i
+end
+
+p sql.results
+
+# Only select value for key = foo
+# $ env KEY=foo bundle exec ruby examples/sql_add.rb
+# [["bar"]]
+#
+# Only select value for key = bar
+# $ env KEY=bar bundle exec ruby examples/sql_add.rb
+# []
+#
+# Only select value for key = baz
+# $ env KEY=baz bundle exec ruby examples/sql_add.rb
+# [["wick"]]
+#
+# Select all values
+# $ bundle exec ruby examples/sql_add.rb
+# [["bar"], ["wick"]]
+#
+# Select only 1 key.
+# $ env LIMIT=1 bundle exec ruby examples/sql_add.rb
+# [["wick"]]

data/examples/sql_with_connection.rb

@@ -0,0 +1,32 @@
+require File.expand_path("../example_setup", __FILE__)
+require "github/sql"
+
+class SomeModel < ActiveRecord::Base
+  self.abstract_class = true
+
+  establish_connection({
+    adapter: "mysql2",
+    database: "github_ds_test",
+  })
+end
+
+insert_statement = "INSERT INTO example_key_values (`key`, `value`) VALUES (:key, :value)"
+
+ActiveRecord::Base.transaction do
+  # Insert bar on base connection.
+  GitHub::SQL.run insert_statement, key: "bar", value: "baz", connection: ActiveRecord::Base.connection
+
+  SomeModel.transaction do
+    # Insert foo on different connection.
+    GitHub::SQL.run insert_statement, key: "foo", value: "bar", connection: SomeModel.connection
+  end
+
+  # Roll back "bar" insertion.
+  raise ActiveRecord::Rollback
+end
+
+# Show that "bar" key is not here because that connection's transaction was
+# rolled back. SomeModel is a different connection and started a different
+# transaction, which succeeded, so "foo" key was created.
+p GitHub::SQL.values "SELECT `key` FROM example_key_values"
+# ["foo"]

data/github-ds.gemspec

@@ -0,0 +1,42 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "github/ds/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "github-ds"
+  spec.version       = GitHub::DS::VERSION
+  spec.authors       = ["GitHub Open Source", "John Nunemaker"]
+  spec.email         = ["opensource+github-ds@github.com", "nunemaker@gmail.com"]
+
+  spec.summary       = %q{A collection of libraries for working with SQL on top of ActiveRecord's connection.}
+  spec.description   = %q{A collection of libraries for working with SQL on top of ActiveRecord's connection.}
+  spec.homepage      = "https://github.com/github/github-ds"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "timecop", "~> 0.8.1"
+  spec.add_development_dependency "activerecord", "~> 5.0"
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "mysql2"
+  spec.add_development_dependency "activerecord-mysql-adapter"
+  spec.add_development_dependency "mocha", "~> 1.2.1"
+end

data/lib/generators/github/ds/active_record_generator.rb

@@ -0,0 +1,22 @@
+require "rails/generators/active_record"
+
+module Github
+  module DS
+    module Generators
+      class ActiveRecordGenerator < ::Rails::Generators::Base
+        include ::Rails::Generators::Migration
+        desc "Generates migration for KV table"
+
+        source_paths << File.join(File.dirname(__FILE__), "templates")
+
+        def create_migration_file
+          migration_template "migration.rb", "db/migrate/create_key_values_table.rb"
+        end
+
+        def self.next_migration_number(dirname)
+          ::ActiveRecord::Generators::Base.next_migration_number(dirname)
+        end
+      end
+    end
+  end
+end

data/lib/generators/github/ds/templates/migration.rb

@@ -0,0 +1,19 @@
+class CreateKeyValuesTable < ActiveRecord::Migration
+  def self.up
+    create_table :key_values do |t|
+      t.string :key, :null => false
+      t.binary :value, :null => false
+      t.datetime :expires_at, :null => true
+      t.timestamps :null => false
+    end
+
+    add_index :key_values, :key, :unique => true
+    add_index :key_values, :expires_at
+
+    change_column :key_values, :id, "bigint(20) NOT NULL AUTO_INCREMENT"
+  end
+
+  def self.down
+    drop_table :key_values
+  end
+end

data/lib/github-ds.rb

@@ -0,0 +1 @@
+require "github/ds"

data/lib/github/ds.rb

@@ -0,0 +1,8 @@
+require "github/ds/version"
+
+module GitHub
+  module DS
+  end
+end
+
+require "github/kv"

data/lib/github/ds/version.rb

@@ -0,0 +1,5 @@
+module GitHub
+  module DS
+    VERSION = "0.1.0"
+  end
+end

data/lib/github/kv.rb

@@ -0,0 +1,343 @@
+require "github/result"
+require "github/sql"
+
+# GitHub::KV is a key/value data store backed by MySQL (however, the backing
+# store used should be regarded as an implementation detail).
+#
+# Usage tips:
+#
+#   * Components in key names should be ordered by cardinality, from lowest to
+#     highest. That is, static key components should be at the front of the key
+#     and key components that vary should be at the end of the key in order of
+#     how many potential values they might have.
+#
+#     For example, if using GitHub::KV to store a user preferences, the key
+#     should be named "user.#{preference_name}.#{user_id}". Notice that the
+#     part of the key that never changes ("user") comes first, followed by
+#     the name of the preference (of which there might be a handful), followed
+#     finally by the user id (of which there are millions).
+#
+#     This will make it easier to scan for keys later on, which is a necessity
+#     if we ever need to move this data out of GitHub::KV or if we need to
+#     search the keyspace for some reason (for example, if it's a preference
+#     that we're planning to deprecate, putting the preference name near the
+#     beginning of the key name makes it easier to search for all users with
+#     that preference set).
+#
+#   * All reader methods in GitHub::KV return values wrapped inside a Result
+#     object.
+#
+#     If any of these methods raise an exception for some reason (for example,
+#     the database is down), they will return a Result value representing this
+#     error rather than raising the exception directly. See lib/github/result.rb
+#     for more documentation on GitHub::Result including usage examples.
+#
+#     When using GitHub::KV, it's important to handle error conditions and not
+#     assume that GitHub::Result objects will always represent success.
+#     Code using GitHub::KV should be able to fail partially if
+#     GitHub::KV is down. How exactly to do this will depend on a
+#     case-by-case basis - it may involve falling back to a default value, or it
+#     might involve showing an error message to the user while still letting the
+#     rest of the page load.
+#
+module GitHub
+  class KV
+    MAX_KEY_LENGTH = 255
+    MAX_VALUE_LENGTH = 65535
+
+    KeyLengthError = Class.new(StandardError)
+    ValueLengthError = Class.new(StandardError)
+    UnavailableError = Class.new(StandardError)
+
+    class MissingConnectionError < StandardError; end
+
+    def initialize(encapsulated_errors = [SystemCallError], &conn_block)
+      @encapsulated_errors = encapsulated_errors
+      @conn_block = conn_block
+    end
+
+    def connection
+      @conn_block.try(:call) || (raise MissingConnectionError, "KV must be initialized with a block that returns a connection")
+    end
+
+    # get :: String -> Result<String | nil>
+    #
+    # Gets the value of the specified key.
+    #
+    # Example:
+    #
+    #   kv.get("foo")
+    #     # => #<Result value: "bar">
+    #
+    #   kv.get("octocat")
+    #     # => #<Result value: nil>
+    #
+    def get(key)
+      validate_key(key)
+
+      mget([key]).map { |values| values[0] }
+    end
+
+    # mget :: [String] -> Result<[String | nil]>
+    #
+    # Gets the values of all specified keys. Values will be returned in the
+    # same order as keys are specified. nil will be returned in place of a
+    # String for keys which do not exist.
+    #
+    # Example:
+    #
+    #   kv.mget(["foo", "octocat"])
+    #     # => #<Result value: ["bar", nil]
+    #
+    def mget(keys)
+      validate_key_array(keys)
+
+      Result.new {
+        kvs = GitHub::SQL.results(<<-SQL, :keys => keys, :connection => connection).to_h
+          SELECT `key`, value FROM key_values WHERE `key` IN :keys AND (`expires_at` IS NULL OR `expires_at` > NOW())
+        SQL
+
+        keys.map { |key| kvs[key] }
+      }
+    end
+
+    # set :: String, String, expires: Time? -> nil
+    #
+    # Sets the specified key to the specified value. Returns nil. Raises on
+    # error.
+    #
+    # Example:
+    #
+    #   kv.set("foo", "bar")
+    #     # => nil
+    #
+    def set(key, value, expires: nil)
+      validate_key(key)
+      validate_value(value)
+
+      mset({ key => value }, expires: expires)
+    end
+
+    # mset :: { String => String }, expires: Time? -> nil
+    #
+    # Sets the specified hash keys to their associated values, setting them to
+    # expire at the specified time. Returns nil. Raises on error.
+    #
+    # Example:
+    #
+    #   kv.mset({ "foo" => "bar", "baz" => "quux" })
+    #     # => nil
+    #
+    #   kv.mset({ "expires" => "soon" }, expires: 1.hour.from_now)
+    #     # => nil
+    #
+    def mset(kvs, expires: nil)
+      validate_key_value_hash(kvs)
+      validate_expires(expires) if expires
+
+      rows = kvs.map { |key, value|
+        [key, value, GitHub::SQL::NOW, GitHub::SQL::NOW, expires || GitHub::SQL::NULL]
+      }
+
+      encapsulate_error do
+        GitHub::SQL.run(<<-SQL, :rows => GitHub::SQL::ROWS(rows), :connection => connection)
+          INSERT INTO key_values (`key`, value, created_at, updated_at, expires_at)
+          VALUES :rows
+          ON DUPLICATE KEY UPDATE
+            value = VALUES(value),
+            updated_at = VALUES(updated_at),
+            expires_at = VALUES(expires_at)
+        SQL
+      end
+
+      nil
+    end
+
+    # exists :: String -> Result<Boolean>
+    #
+    # Checks for existence of the specified key.
+    #
+    # Example:
+    #
+    #   kv.exists("foo")
+    #     # => #<Result value: true>
+    #
+    #   kv.exists("octocat")
+    #     # => #<Result value: false>
+    #
+    def exists(key)
+      validate_key(key)
+
+      mexists([key]).map { |values| values[0] }
+    end
+
+    # mexists :: [String] -> Result<[Boolean]>
+    #
+    # Checks for existence of all specified keys. Booleans will be returned in
+    # the same order as keys are specified.
+    #
+    # Example:
+    #
+    #   kv.mexists(["foo", "octocat"])
+    #     # => #<Result value: [true, false]>
+    #
+    def mexists(keys)
+      validate_key_array(keys)
+
+      Result.new {
+        existing_keys = GitHub::SQL.values(<<-SQL, :keys => keys, :connection => connection).to_set
+          SELECT `key` FROM key_values WHERE `key` IN :keys AND (`expires_at` IS NULL OR `expires_at` > NOW())
+        SQL
+
+        keys.map { |key| existing_keys.include?(key) }
+      }
+    end
+
+    # setnx :: String, String, expires: Time? -> Boolean
+    #
+    # Sets the specified key to the specified value only if it does not
+    # already exist.
+    #
+    # Returns true if the key was set, false otherwise. Raises on error.
+    #
+    # Example:
+    #
+    #   kv.setnx("foo", "bar")
+    #     # => false
+    #
+    #   kv.setnx("octocat", "monalisa")
+    #     # => true
+    #
+    #   kv.setnx("expires", "soon", expires: 1.hour.from_now)
+    #     # => true
+    #
+    def setnx(key, value, expires: nil)
+      validate_key(key)
+      validate_value(value)
+      validate_expires(expires) if expires
+
+      encapsulate_error {
+        # if the key already exists but has expired, prune it first. We could
+        # achieve the same thing with the right INSERT ... ON DUPLICATE KEY UPDATE
+        # query, but then we would not be able to rely on affected_rows
+
+        GitHub::SQL.run(<<-SQL, :key => key, :connection => connection)
+          DELETE FROM key_values WHERE `key` = :key AND expires_at <= NOW()
+        SQL
+
+        sql = GitHub::SQL.run(<<-SQL, :key => key, :value => value, :expires => expires || GitHub::SQL::NULL, :connection => connection)
+          INSERT IGNORE INTO key_values (`key`, value, created_at, updated_at, expires_at)
+          VALUES (:key, :value, NOW(), NOW(), :expires)
+        SQL
+
+        sql.affected_rows > 0
+      }
+    end
+
+    # del :: String -> nil
+    #
+    # Deletes the specified key. Returns nil. Raises on error.
+    #
+    # Example:
+    #
+    #   kv.del("foo")
+    #     # => nil
+    #
+    def del(key)
+      validate_key(key)
+
+      mdel([key])
+    end
+
+    # mdel :: String -> nil
+    #
+    # Deletes the specified keys. Returns nil. Raises on error.
+    #
+    # Example:
+    #
+    #   kv.mdel(["foo", "octocat"])
+    #     # => nil
+    #
+    def mdel(keys)
+      validate_key_array(keys)
+
+      encapsulate_error do
+        GitHub::SQL.run(<<-SQL, :keys => keys, :connection => connection)
+          DELETE FROM key_values WHERE `key` IN :keys
+        SQL
+      end
+
+      nil
+    end
+
+  private
+    def validate_key(key)
+      raise TypeError, "key must be a String in #{self.class.name}, but was #{key.class}" unless key.is_a?(String)
+
+      validate_key_length(key)
+    end
+
+    def validate_value(value)
+      raise TypeError, "value must be a String in #{self.class.name}, but was #{value.class}" unless value.is_a?(String)
+
+      validate_value_length(value)
+    end
+
+    def validate_key_array(keys)
+      unless keys.is_a?(Array)
+        raise TypeError, "keys must be a [String] in #{self.class.name}, but was #{keys.class}"
+      end
+
+      keys.each do |key|
+        unless key.is_a?(String)
+          raise TypeError, "keys must be a [String] in #{self.class.name}, but also saw at least one #{key.class}"
+        end
+
+        validate_key_length(key)
+      end
+    end
+
+    def validate_key_value_hash(kvs)
+      unless kvs.is_a?(Hash)
+        raise TypeError, "kvs must be a {String => String} in #{self.class.name}, but was #{key.class}"
+      end
+
+      kvs.each do |key, value|
+        unless key.is_a?(String)
+          raise TypeError, "kvs must be a {String => String} in #{self.class.name}, but also saw at least one key of type #{key.class}"
+        end
+
+        unless value.is_a?(String)
+          raise TypeError, "kvs must be a {String => String} in #{self.class.name}, but also saw at least one value of type #{value.class}"
+        end
+
+        validate_key_length(key)
+        validate_value_length(value)
+      end
+    end
+
+    def validate_key_length(key)
+      if key.length > MAX_KEY_LENGTH
+        raise KeyLengthError, "key of length #{key.length} exceeds maximum key length of #{MAX_KEY_LENGTH}\n\nkey: #{key.inspect}"
+      end
+    end
+
+    def validate_value_length(value)
+      if value.length > MAX_VALUE_LENGTH
+        raise ValueLengthError, "value of length #{value.length} exceeds maximum value length of #{MAX_VALUE_LENGTH}"
+      end
+    end
+
+    def validate_expires(expires)
+      unless expires.respond_to?(:to_time)
+        raise TypeError, "expires must be a time of some sort (Time, DateTime, ActiveSupport::TimeWithZone, etc.), but was #{expires.class}"
+      end
+    end
+
+    def encapsulate_error
+      yield
+    rescue *@encapsulated_errors => error
+      raise UnavailableError, "#{error.class}: #{error.message}"
+    end
+  end
+end