octoshark 0.1.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f8eac1fa769be042b9c258530c49c05ef91723ab
-  data.tar.gz: 1852cb828c3babdcdcb79298e4619581bb72b216
+SHA256:
+  metadata.gz: 743be64d2aeb897827aec871dfd86150ea1ca7ce6505839cea1caf16e85676bc
+  data.tar.gz: a801773571a7388fed3d16179be6006d10a6e060fb514813a2e21ba3532816ee
 SHA512:
-  metadata.gz: 535e065d62a6cd01e38d6d993e25a02f182a0dfa07df4b3150de13418caabeaf325531bd3560005b62e6e002e39b2835fcea701621d5454c03e01a16916cd325
-  data.tar.gz: 026dee2bd0e09b7f5ea1c3c2300727ff224f6820e51cc562cec57bd33ad92765267e760957168841054dcb76fd0391d19c4dfc99588092c1c49fdd7487887a39
+  metadata.gz: 2fa8e793d0eae00f5d99420e1f3b5bc6218b9b5a5b8c27b0c63dbbde080265d5797cc4d56cbce049cd25459e0f8961228f0acb5d677f35f1de45a42fbd82f610
+  data.tar.gz: 16f82b5d43d619ff5d8691229a8377189c10ef3b9d20752649d2e63725bff171afe9f232d2300916b573bff7260ff63d92be08bae567934ff7751285d6240728

data/.github/workflows/ci-legacy.yml

@@ -0,0 +1,66 @@
+name: Build Legacy
+
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-16.04
+    services:
+     mysql:
+        image: mysql:5.5
+        env:
+          MYSQL_ROOT_PASSWORD: pass
+        ports:
+          - "3306:3306"
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    name: ruby-${{ matrix.ruby }} ${{ matrix.gemfile }}
+    strategy:
+      matrix:
+        include:
+          - gemfile: rails3.0
+            ruby: 2.2
+          - gemfile: rails3.1
+            ruby: 2.2
+          - gemfile: rails3.2
+            ruby: 2.2
+
+          - gemfile: rails4.0
+            ruby: 2.4
+          - gemfile: rails4.1
+            ruby: 2.4
+          - gemfile: rails4.2
+            ruby: 2.4
+
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
+      BUNDLE_PATH_RELATIVE_TO_CWD: true
+
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler: default
+          bundler-cache: true
+
+      - name: Set up database
+        run: |
+          cp spec/support/config.yml.github spec/support/config.yml
+          bundle exec rake db:create
+
+      - name: Run tests
+        run: |
+          bundle exec rspec spec

data/.github/workflows/ci.yml

@@ -0,0 +1,64 @@
+name: Build
+
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+     mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: pass
+        ports:
+          - "3306:3306"
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    name: ruby-${{ matrix.ruby }} ${{ matrix.gemfile }}
+    strategy:
+      matrix:
+        include:
+          - gemfile: rails5.0
+            ruby: 2.6
+          - gemfile: rails5.1
+            ruby: 2.6
+          - gemfile: rails5.2
+            ruby: 2.6
+
+          - gemfile: rails6.0
+            ruby: 2.7
+          - gemfile: rails6.1
+            ruby: 2.7
+
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
+      BUNDLE_PATH_RELATIVE_TO_CWD: true
+
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler: default
+          bundler-cache: true
+
+      - name: Set up database
+        run: |
+          cp spec/support/config.yml.github spec/support/config.yml
+          bundle exec rake db:create
+
+      - name: Run tests
+        run: |
+          bundle exec rspec spec

data/.ruby-version

@@ -1 +1 @@
-2.1.1
+2.7.4

data/Appraisals

@@ -1,19 +1,29 @@
-appraise "rails3.1" do
-  gem "activerecord", "~> 3.1.0"
+appraise "rails5.0" do
+  gem "activerecord", "~> 5.0.0"
+  gem "mysql2", "~> 0.5.2"
+  gem "sqlite3",  "~> 1.3.13"
 end
 
-appraise "rails3.2" do
-  gem "activerecord", "~> 3.2.0"
+appraise "rails5.1" do
+  gem "activerecord", "~> 5.1.0"
+  gem "mysql2", "~> 0.5.2"
+  gem "sqlite3",  "~> 1.4.1"
 end
 
-appraise "rails4" do
-  gem "activerecord", "~> 4.0.0"
+appraise "rails5.2" do
+  gem "activerecord", "~> 5.2.0"
+  gem "mysql2", "~> 0.5.2"
+  gem "sqlite3",  "~> 1.4.1"
 end
 
-appraise "rails4.1" do
-  gem "activerecord", "~> 4.1.0"
+appraise "rails6.0" do
+  gem "activerecord", "~> 6.0.0"
+  gem "mysql2", "~> 0.5.2"
+  gem "sqlite3",  "~> 1.4.1"
 end
 
-appraise "rails4.2" do
-  gem "activerecord", "~> 4.2.0"
+appraise "rails6.1" do
+  gem "activerecord", "~> 6.1.0"
+  gem "mysql2", "~> 0.5.2"
+  gem "sqlite3",  "~> 1.4.1"
 end

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Change log
+
+## 0.3.0 2021-07-20
+
+- Add support for Rails 6.1
+
+## 0.2.2 2016-11-09
+
+- Fix `alias_method_chain` deprecation in Rails 5
+
+## 0.2.1 2016-09-09
+
+- Add support for Rails 5 and Rails 3
+
+## 0.2.0 2016-08-29
+
+- `Octoshark::ConnectionManager` is split in two managers `Octoshark::ConnectionPoolsManager` for persistent connections and `Octoshark::ConnectionManager` for non-persistent connections.
+- `Octoshark` class methods like `connection_managers`, `reset_connection_managers!`, `disconnect!` are only relevant and moved to `Octoshark::ConnectionPoolsManager` class.
+- `Octoshark::ConnectionManager#use_database` method has been removed and the functionality moved to `Octoshark::ConnectionPoolsManager#with_connection(name, database_name)` where the second optional argument `database_name` when specified will switch the connection to the database using the `use database` MySQL statement.

data/Gemfile

@@ -1,4 +1,5 @@
 source 'https://rubygems.org'
+gem "activerecord", "~> 6.1.0"
 
 # Specify your gem's dependencies in octoshark.gemspec
 gemspec

data/README.md

@@ -1,8 +1,8 @@
-![Octoshark logo](https://dl.dropboxusercontent.com/u/3230730/github/octoshark.png)
+![Octoshark logo](http://dalibornasevic.com/images/octoshark.png)
 
 ![Travis status](https://travis-ci.org/dalibor/octoshark.png)
 
-Octoshark is a simple ActiveRecord connection manager. It provides a connection switching mechanism that can be used in various scenarios like: sharding, master-slave, multi-tenancy, etc. You have control over how connections are configured and used, see below for examples.
+Octoshark is a simple ActiveRecord connection manager. It provides connection switching mechanisms that can be used in various scenarios like master-slave, sharding or multi-tenant architecture.
 
 
 ## Installation
@@ -28,138 +28,142 @@ $ gem install octoshark
 
 ## Usage
 
-Create a new connection manager with connection 2 pools for Octoshark to manage. This usually goes in the app initializer.
+Octoshark has two connection managers: `ConnectionPoolsManager` for managing connection pools using persistent connections and `ConnectionManager` for managing non-persistent connections. It depends on your application performance and scaling requirements which one to use.
 
-```ruby
-CONN_MANAGER = Octoshark::ConnectionManager.new({
-  db1: { adapter: "sqlite3", database: "db/db1.sqlite" },
-  db2: { adapter: "sqlite3", database: "db/db2.sqlite" }
-})
-```
+- If you have a limited number of consumers (application and worker servers), `ConnectionPoolsManager` would be the preferred option. Standard Rails application has a single connection pool, and `ConnectionPoolsManager` just makes it possible for application models to work with multiple connection pools.
+
+- If you have a very big infrastructure with lots of consumers (application and worker servers) and you are hitting max connections limit on database servers, i.e. you need to scale horizontally, `ConnectionManager` is the option to use. Because it uses non-persistent connections it comes up with a performance penalty because connections are re-established over and over again. Some ActiveRecord plugins that depend on having an active database connection all the time might need a change in order to work with non-persistent connections.
+
+`ConnectionPoolsManager` and `ConnectionManager` can be combined together and many of them can be used at the same time.
 
-Configure which ActiveRecord model will use the Octoshark connection by overriding the `Model.connection` method.
+Here is how to create connection pools manager:
 
 ```ruby
-class Post < ActiveRecord::Base
-  def self.connection
-    CONN_MANAGER.current_connection
-  end
-end
+CONN_MANAGER = Octoshark::ConnectionPoolsManager.new({ c1: config1, c2: config2 })
 ```
 
-Alternatively, extract it as a module and include in models.
+`config1` and `config2` are standard ActiveRecord database configs:
 
 ```ruby
-module ShardingModel
-  extend ActiveSupport::Concern
-
-  module ClassMethods
-    def connection
-      OCTOSHARK.current_connection
-    end
-  end
-end
+config = {
+  adapter:  'mysql2',
+  host:     'localhost',
+  port:     3306,
+  database: 'database',
+  username: 'root',
+  password: 'pass',
+  pool:     1,
+  encoding: 'utf8',
+  reconnect: false
+}
 ```
 
-To use a specific database connection:
+To switch a connection using a specific pool:
 
 ```ruby
-CONN_MANAGER.with_connection(:db1) do
-  # run queries on db1
-  Post.first
+CONN_MANAGER.with_connection(:c1) do |connection|
+  connection.execute("SELECT 1")
 end
 ```
 
 Multiple `with_connection` blocks can be nested:
 
 ```ruby
-CONN_MANAGER.with_connection(:db1) do
-  # run queries on db1
+CONN_MANAGER.with_connection(config1) do
+  # run queries on connection specified with config1
 
-  CONN_MANAGER.with_connection(:db2) do
-    # run queries on db2
+  CONN_MANAGER.with_connection(config2) do
+    # run queries on connection specified with config2
   end
 
-  # run queries on db1
+  # run queries on connection specified with config1
 end
 ```
 
-`CONN_MANAGER.current_connection` returns the active connection while in the `with_connection` block or raises `Octoshark::Error::NoCurrentConnection` otherwise.
+If you establish a connection to database server instead of to database, then you can use the second optional argument `database_name` to tell the connection manager to switch the connection to that database within the same connection. This is useful when you want to have fewer connection pools and keep number of active connection per database server under control. This option is only MySQL specific for now, it uses `USE database_name` statement to switch the connection.
 
+```ruby
+CONN_MANAGER.with_connection(:c1, database_name) do |connection|
+  connection.execute("SELECT 1")
+end
+```
 
-## Sharding example
-
-Some models are in the core DB, and others in shard DBs. Shard is selected based on a user attribute. For core models use the default ActiveRecord connection and for sharded models define and use Octoshark connections.
+Using non-persistent connections with `Octoshark::ConnectionManager` has the same API:
 
-Switch the connection in a controller with an around filter:
 
 ```ruby
-around_filter :select_shard
+CONN_MANAGER = Octoshark::ConnectionManager.new
+```
 
-def select_shard(&block)
-  CONN_MANAGER.with_connection(current_user.shard, &block)
+Opening a new connection, executing query and closing the connection:
+
+```ruby
+CONN_MANAGER.with_connection(config) do |connection|
+  connection.execute("SELECT 1")
 end
 ```
 
-Similar approach applies to other application entry-points like background jobs.
-
 
-## Master-Slave example
+## Using Octoshark with ActiveRecord models
 
-All models are in master and slave databases. For master models use the default ActiveRecord connection and for slave models define and use Octoshark connections.
+To tell an ActiveRecord model to use the Octoshark connection we can override the `Model.connection` method.
 
 ```ruby
-class ActiveRecord::Base
+class Post < ActiveRecord::Base
   def self.connection
-    # Return the current connection (from with_connection block) or default one
-    CONN_MANAGER.current_or_default_connection
+    CONN_MANAGER.current_connection
   end
 end
 ```
 
-`CONN_MANAGER.current_or_default_connection` method returns the current connection while in `with_connection` block or the default ActiveRecord connection when outside.
-
+Alternatively, we can extract it as a module and include in multiple models.
 
-## Multi-tenant example
+```ruby
+module ShardingModel
+  extend ActiveSupport::Concern
 
-Some models are in the core DB, and others in user's own dedicated database. For core models use the default ActiveRecord connection and for tenant models can use Octoshark's mechanism to create new temporary connection.
+  module ClassMethods
+    def connection
+      CONN_MANAGER.current_connection
+    end
+  end
+end
+```
 
-Switch the connection in a controller with an around filter:
+To use a specific database connection:
 
 ```ruby
-# in initializer
-CONN_MANAGER = Octoshark::ConnectionManager.new
+CONN_MANAGER.with_connection(:c1) do
+  # run queries on c1
+  Post.first
+end
+```
+
+This connection switching in Rails applications is usually done from within an `around_filter` for controllers and in a similar way for other application "entry-points" like background jobs:
 
-# in controller
+```ruby
 around_filter :select_shard
 
 def select_shard(&block)
-  CONN_MANAGER.with_new_connection(name, config, reusable: false, &block)
+  CONN_MANAGER.with_connection(current_user.shard, &block)
 end
 ```
 
-`CONN_MANAGER.with_new_connection` method creates a temporary connection that will automatically disconnect. If you want to reuse it in subsequent connection switches, set `reusable: true` and it will be added to the connection manager and reused with the next calls. Depends on the use-case and what's preferable. In test environment usually you would want to set it to `reusable` so that database cleaner can clean data with transaction strategy.
+`CONN_MANAGER.current_connection` returns the active connection while the execution is in the `with_connection` block or raises `Octoshark::Error::NoCurrentConnection` outside of the `with_connection` block. In some cases, falling back to the default database connection for the Rails app might be preferable which can be done using `CONN_MANAGER.current_or_default_connection`.
 
-Alternatively, for better performance (only supported on MySQL), database connection can be switched with `use database` statement. Once connection manager is defined with connectino configs to database servers, selecting a database can be done with:
 
-```ruby
-CONN_MANAGER.use_database(:db1, 'database') do
-  # run queries on database server identified by 'db1' using database 'database'
-end
-```
+## Octoshark::ConnectionPoolsManager.reset_connection_managers!
 
-## Octoshark.reset_connection_managers!
-
-Whenever ActiveRecord::Base calls `establish_connection` (usually by an ancestor process that must have subsequently forked), `Octoshark.reset_connection_managers!` is automatically called to re-establish the Octoshark connections. It prevents `ActiveRecord::ConnectionNotEstablished` in the scenarios like:
+When using `Octoshark::ConnectionPoolsManager`, whenever ActiveRecord::Base calls `establish_connection` (usually by an ancestor process that must have subsequently forked), `Octoshark.reset_connection_managers!` is automatically called to re-establish the Octoshark connections. It prevents `ActiveRecord::ConnectionNotEstablished` in the scenarios like:
 
 * Unicorn before/after fork
 * Spring prefork/serve
 * Some rake tasks like `rake db:test:prepare`
 
 
-## Database Cleaner
+## Cleaning test databases
 
-RSpec example on how to clean default and Octoshark data with Database Cleaner:
+For `Octoshark::ConnectionPoolsManager`, we can use [DatabaseCleaner](https://github.com/DatabaseCleaner/database_cleaner) and RSpec like:
 
 ```ruby
 config.before(:suite) do
@@ -173,13 +177,12 @@ config.before(:each) do
 end
 
 config.after(:each) do
-  setup_database_cleaner
   DatabaseCleaner.clean_with(:transaction)
 end
 
 def setup_database_cleaner
   DatabaseCleaner[:active_record, {connection: ActiveRecord::Base.connection_pool}]
-  Octoshark.connection_managers.each do |manager|
+  Octoshark::ConnectionPoolsManager.connection_managers.each do |manager|
     manager.connection_pools.each_pair do |connection_name, connection_pool|
       DatabaseCleaner[:active_record, {connection: connection_pool}]
     end
@@ -187,6 +190,118 @@ def setup_database_cleaner
 end
 ```
 
+For `Octoshark::ConnectionManager` where connections and databases are dynamically created and cannot be configured in the test setup, we can write a custom database cleaner inspired by [DatabaseRewinder](https://github.com/amatsuda/database_rewinder). The example below is for a multi-tenant test setup with a main (core) database and a tenant database for the `CURRENT_USER` in the test suite. `CustomDatabaseCleaner.clean_all` cleans all core database tables before test suite and `CustomDatabaseCleaner.clean` cleans used tables in both core and tenant databases after each test.
+
+
+```ruby
+module CustomDatabaseCleaner
+  INSERT_REGEX = /\AINSERT(?:\s+IGNORE)?\s+INTO\s+(?:\.*[`"]?(?<table>[^.\s`"]+)[`"]?)*/i
+
+  @@tables_with_inserts = []
+
+  class << self
+    def record_inserted_table(connection, sql)
+      match = sql.match(INSERT_REGEX)
+
+      if match && match[:table] && tables_with_inserts.exclude?(match[:table])
+        tables_with_inserts << match[:table]
+      end
+    end
+
+    def clean_all
+      with_core_db_connection do |connection|
+        clean_tables(connection)
+      end
+
+      reset_tables_with_inserts
+    end
+
+    def clean
+      with_core_db_connection do |connection|
+        clean_tables(connection, { 'users' => [CURRENT_USER.id] })
+      end
+
+      CURRENT_USER.with_tenant do |connection|
+        clean_tables(connection)
+      end
+
+      reset_tables_with_inserts
+    end
+
+    private
+    def with_core_db_connection(&block)
+      CoreDBManager.with_connection(ActiveRecord::Base.configurations[Rails.env].symbolize_keys, &block)
+    end
+
+    def clean_tables(connection, keep_data = {})
+      tables_to_clean = connection.tables.reject { |t| t == ActiveRecord::Migrator.schema_migrations_table_name }
+      tables_to_clean = tables_to_clean & tables_with_inserts if tables_with_inserts.present?
+
+      tables_to_clean.each do |table|
+        connection.disable_referential_integrity do
+          table_name = connection.quote_table_name(table)
+          keep_ids   = keep_data[table]
+
+          if keep_ids
+            connection.execute("DELETE FROM #{table_name} WHERE id NOT IN (#{keep_ids.join(',')});")
+          else
+            connection.execute("DELETE FROM #{table_name};")
+          end
+        end
+      end
+    end
+
+    def reset_tables_with_inserts
+      @@tables_with_inserts = []
+    end
+
+    def tables_with_inserts
+      @@tables_with_inserts
+    end
+  end
+end
+
+module CustomDatabaseCleaner
+  module InsertRecorder
+    def execute(sql, *)
+      CustomDatabaseCleaner.record_inserted_table(self, sql)
+      super
+    end
+
+    def exec_query(sql, *)
+      CustomDatabaseCleaner.record_inserted_table(self, sql)
+      super
+    end
+  end
+end
+
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.send(:prepend, CustomDatabaseCleaner::InsertRecorder)
+```
+
+
+## Development Setup
+
+Setup database config and create databases:
+
+```bash
+cp spec/support/config.yml.template spec/support/config.yml
+rake db:create
+```
+
+Run specs:
+
+```bash
+bundle exec rspec spec
+```
+
+Install different active record versions defined in `Appraisals` and run specs for all of them:
+
+```bash
+bundle exec appraisal
+bundle exec appraisal rspec spec
+```
+
 
 ## Logo