async-mysql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b996e0d3b933ddc339252f9a4aa516eedde0802f6176ae56715e2de78a6ff671
+  data.tar.gz: 83b7be26aaa4c6208ca589d08bd37bef7e31bba65e46acca3c65cacd86b07d4b
+SHA512:
+  metadata.gz: cf4eecdf457fb171f8c2e8fca944f3a63934640c7cf013edd6c0578e0b0c7beb7f06e2bea6522e9784e59483621c93d6c8053586c6719e9a57ca187cee10e218
+  data.tar.gz: b2d4c8553aa2c4e456793d23bc6d1832b3afa85434185b9b61fcbdd65b6d1a7e5278f535f032e7021fcafbaa93ea944db8914d139be52b1df5ed35b2d482ba6b

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+/Gemfile.lock

data/.rspec

@@ -0,0 +1,4 @@
+--format documentation
+--backtrace
+--warnings
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,21 @@
+language: ruby
+sudo: false
+dist: trusty
+cache: bundler
+services:
+  - mysqlql
+before_script:
+  - psql -c 'create database test;' -U mysql
+rvm:
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - jruby-head
+  - ruby-head
+  - rbx-3
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-head
+    - rvm: rbx-3

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem "pg", "~> 0.18"
+
+group :development do
+	gem 'pry'
+end
+
+group :test do
+	gem 'simplecov'
+	gem 'coveralls', require: false
+end

data/README.md

@@ -0,0 +1,65 @@
+# Async::MySQL
+
+This is an experimental drop in wrapper to make MySQL work asynchronously, including an ActiveRecord connection adapter.
+
+## Motivation
+
+We have some IO bound web APIs generating statistics and we sometimes have issues when using [passenger] due to thread/process exhaustion. In addition, we make a lot of upstream HTTP RPCs and these are also IO bound.
+
+This library, in combination with [async-http], ensure that we don't become IO bound in many cases. In addition, we don't need to tune the intermediate server as it will simply scale according to backend resource availability and IO throughput.
+
+[passenger]: https://github.com/phusion/passenger
+[async-http]: https://github.com/socketry/async-http
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'async-mysql'
+```
+
+And then execute:
+
+	$ bundle
+
+Or install it yourself as:
+
+	$ gem install async-mysql
+
+## Usage
+
+Use the `async_mysql` adapter, as in `adapter: async_mysql` in your database configuration. This is a drop-in replacement for the MySQL adapter for ActiveRecord. But, it must be used with an [async] capable server like [falcon].
+
+[async]: https://github.com/socketry/async
+[falcon]: https://github.com/socketry/falcon
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Copyright, 2018, by Samuel G. D. Williams. <http://www.codeotaku.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:test)
+
+task :default => :test
+
+task :console do
+	require 'pry'
+	
+	require_relative 'lib/ffi/mysql'
+	
+	Pry.start
+end

data/async-mysql.gemspec

@@ -0,0 +1,27 @@
+
+require_relative 'lib/async/mysql/version'
+
+Gem::Specification.new do |spec|
+	spec.name          = "async-mysql"
+	spec.version       = Async::MySQL::VERSION
+	spec.authors       = ["Samuel Williams"]
+	spec.email         = ["samuel.williams@oriontransfer.co.nz"]
+	spec.summary       = %q{Access mysql without blocking.}
+	spec.homepage      = "https://github.com/socketry/async-mysql"
+	spec.license       = "MIT"
+
+	spec.files         = `git ls-files`.split($/)
+	spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+	spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+	spec.require_paths = ["lib"]
+
+	spec.add_dependency "async", "~> 1.3"
+	spec.add_dependency "mysql2"
+
+	spec.add_development_dependency "async-rspec"
+	spec.add_development_dependency "activerecord"
+
+	spec.add_development_dependency "bundler", "~> 1.3"
+	spec.add_development_dependency "rspec", "~> 3.6"
+	spec.add_development_dependency "rake"
+end

data/examples/rack/config.ru

@@ -0,0 +1,26 @@
+
+$LOAD_PATH << "../../lib"
+
+gem 'mysql2'
+require 'active_record'
+
+if defined?(Falcon)
+	$stderr.puts "Loading async/mysql"
+	require_relative '../../lib/async/mysql'
+	
+	ActiveRecord::Base.establish_connection(adapter: "async_mysql", database: "test", pool: 1024)
+else
+	ActiveRecord::Base.establish_connection(adapter: "mysql2", database: "test", pool: 1024)
+end
+
+# ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+run lambda {|env|
+	10.times do
+		ActiveRecord::Base.connection.execute("SELECT SLEEP(0.01)")
+	end
+	
+	ActiveRecord::Base.clear_active_connections!
+	
+	[200, {}, ["Hello World"]]
+}

data/lib/active_record/connection_adapters/async_mysql_adapter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/mysql2_adapter"
+
+module ActiveRecord
+	module ConnectionHandling
+		def async_mysql_connection(config)
+			config = config.symbolize_keys
+			
+			if config[:flags].kind_of? Array
+				config[:flags].push "FOUND_ROWS".freeze
+			else
+				config[:flags] ||= 0
+				config[:flags] |= Mysql2::Client::FOUND_ROWS
+			end
+			
+			client = Async::MySQL::Client.new(config)
+			ConnectionAdapters::Mysql2Adapter.new(client, logger, nil, config)
+		rescue Mysql2::Error => error
+			if error.message.include?("Unknown database")
+				raise ActiveRecord::NoDatabaseError
+			else
+				raise
+			end
+		end
+	end
+end

data/lib/async/mysql.rb

@@ -0,0 +1,22 @@
+# Copyright, 2018, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require_relative "mysql/version"
+require_relative "mysql/client"

data/lib/async/mysql/client.rb

@@ -0,0 +1,51 @@
+# Copyright, 2018, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'async/wrapper'
+
+require 'mysql2'
+
+module Async
+	module MySQL
+		class Client < Wrapper
+			def initialize(config, reactor = nil)
+				@client = ::Mysql2::Client.new(config)
+				
+				super(::IO.for_fd(@client.socket), reactor)
+			end
+			
+			def query(sql, **options)
+				@client.query(sql, async: true, **options)
+				
+				wait_readable
+				
+				return @client.async_result
+			end
+			
+			def respond_to?(*args)
+				@client.respond_to(*args)
+			end
+			
+			def method_missing(*args)
+				@client.send(*args)
+			end
+		end
+	end
+end

data/lib/async/mysql/pool.rb

@@ -0,0 +1,112 @@
+# Copyright, 2018, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'async/reactor'
+
+module Async
+	class Reactor < Node
+		attr_accessor :mysql_pools
+	end
+	
+	module MySQL
+		class Proxy
+			def initialize(connection_string, task: Task.current)
+				@connection_string = connection_string
+				
+				pools = task.reactor.mysql_pools ||= {}
+				
+				@pool = pools[@connection_string] ||= Pool.new do
+					Connection.new(@connection_string)
+				end
+			end
+			
+			def close
+				@pool.close
+			end
+			
+			def async_exec(*args)
+				@pool.acquire do |connection|
+					connection.async_exec(*args)
+				end
+			end
+			
+			def respond_to?(*args)
+				@pool.acquire do |connection|
+					connection.respond_to?(*args)
+				end
+			end
+			
+			def method_missing(*args, &block)
+				@pool.acquire do |connection|
+					connection.send(*args, &block)
+				end
+			end
+		end
+		
+		# This pool doesn't impose a maximum number of open resources, but it WILL block if there are no available resources and trying to allocate another one fails.
+		class Pool
+			def initialize(&block)
+				@available = []
+				@waiting = []
+				
+				@constructor = block
+			end
+			
+			def acquire
+				resource = wait_for_next_available
+				
+				begin
+					yield resource
+				ensure
+					@available << resource
+					
+					if task = @waiting.pop
+						task.resume
+					end
+				end
+			end
+			
+			def close
+				@available.each(&:close)
+				@available.clear
+			end
+			
+			def wait_for_next_available
+				until resource = next_available
+					@waiting << Fiber.current
+					Task.yield
+				end
+				
+				return resource
+			end
+			
+			def next_available
+				if @available.empty?
+					return @constructor.call # This might fail, which is okay :)
+				else
+					return @available.pop
+				end
+			rescue StandardError
+				$stderr.puts $!.inspect
+				return nil
+			end
+		end
+	end
+end

data/lib/async/mysql/version.rb

@@ -0,0 +1,25 @@
+# Copyright, 2018, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Async
+	module MySQL
+		VERSION = "0.1.0"
+	end
+end

data/spec/activerecord/connection_spec.rb

@@ -0,0 +1,28 @@
+
+require 'active_record'
+
+class Book < ActiveRecord::Base
+end
+
+RSpec.describe ActiveRecord do
+	include_context Async::RSpec::Reactor
+	
+	it "should work using async adapter" do
+		reactor.async do
+			ActiveRecord::Base.establish_connection(adapter: "async_mysql", database: "test")
+			ActiveRecord::Base.logger = Logger.new(STDOUT)
+			
+			ActiveRecord::Schema.define do
+				create_table :books, force: true do |t|
+					t.string :name
+					t.timestamps
+				end
+			end
+			
+			Book.create(name: "How to use a fork")
+			
+			# This closes the underlying connection.
+			ActiveRecord::Base.remove_connection
+		end
+	end
+end

data/spec/async/mysql/client_spec.rb

@@ -0,0 +1,18 @@
+
+require 'async/mysql/client'
+
+RSpec.describe Async::MySQL::Client do
+	include_context Async::RSpec::Reactor
+	
+	let(:connection) {Async::MySQL::Client.new(host: 'localhost', database: 'test')}
+	
+	it "should execute query" do
+		reactor.async do
+			results = connection.query("SELECT 42 AS LIFE")
+		
+			expect(results.each.to_a).to be == [{"LIFE" => 42}]
+			
+			connection.close
+		end
+	end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,30 @@
+
+if ENV['COVERAGE']
+	begin
+		require 'simplecov'
+		
+		SimpleCov.start do
+			add_filter "/spec/"
+		end
+		
+		if ENV['TRAVIS']
+			require 'coveralls'
+			Coveralls.wear!
+		end
+	rescue LoadError
+		warn "Could not load simplecov: #{$!}"
+	end
+end
+
+require "bundler/setup"
+require "async/mysql"
+require 'async/rspec'
+
+RSpec.configure do |config|
+	# Enable flags like --only-failures and --next-failure
+	config.example_status_persistence_file_path = ".rspec_status"
+
+	config.expect_with :rspec do |c|
+		c.syntax = :expect
+	end
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: async-mysql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Samuel Williams
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: async-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- samuel.williams@oriontransfer.co.nz
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- async-mysql.gemspec
+- examples/rack/config.ru
+- lib/active_record/connection_adapters/async_mysql_adapter.rb
+- lib/async/mysql.rb
+- lib/async/mysql/client.rb
+- lib/async/mysql/pool.rb
+- lib/async/mysql/version.rb
+- spec/activerecord/connection_spec.rb
+- spec/async/mysql/client_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/socketry/async-mysql
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Access mysql without blocking.
+test_files:
+- spec/activerecord/connection_spec.rb
+- spec/async/mysql/client_spec.rb
+- spec/spec_helper.rb