viewller 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: a70be24277beee467019aafa835f7dde9822c6c8091c6daea304d0c066254b66
4
+ data.tar.gz: 094ea4748c0162678b2fab8ce8c5a819440d50cd67abb66f2807682b3e032e68
5
+ SHA512:
6
+ metadata.gz: abb858a6da2f2611ca4b8b79a1be8c7c8e5f950631a488ff516ace826404335a69ef3598a54ef4ff18ab8e86274403a9c43f578ec372d0ea61c9e9fe04fa2055
7
+ data.tar.gz: d8a5cc9f7573e52c18fe88ff2a2b480e1559ed235e9368f7c70ff4b3a13353e4cf53bfdd43d05570ee77a26dde0406cb23798fa561dae2ae18c6c6f81d0b6d8b
data/.gitignore ADDED
@@ -0,0 +1,18 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ /log/
11
+ /gemfiles/
12
+ *.bundle
13
+ *.so
14
+ *.o
15
+ *.a
16
+ mkmf.log
17
+ .DS_Store
18
+ .env
data/.gitlab-ci.yml ADDED
@@ -0,0 +1,8 @@
1
+ image: ruby:3.3
2
+
3
+ smoke:
4
+ script:
5
+ - ruby -Ilib test/smoke_test.rb
6
+ - gem build viewller.gemspec
7
+ - gem install --local --ignore-dependencies --no-document --install-dir "$CI_PROJECT_DIR/.gem_install" viewller-*.gem
8
+ - ruby -I"$CI_PROJECT_DIR/.gem_install/gems/viewller-$(ruby -Ilib -rviewller/version -e 'print Viewller::VERSION')/lib" test/smoke_test.rb
data/.rubocop.yml ADDED
@@ -0,0 +1,6 @@
1
+ Metrics/LineLength:
2
+ Max: 99
3
+ Style/AccessModifierIndentation:
4
+ EnforcedStyle: outdent
5
+ Style/Documentation:
6
+ Enabled: false
data/.travis.yml ADDED
@@ -0,0 +1,17 @@
1
+ # This file was auto-generated by the schema_dev tool, based on the data in
2
+ # ./schema_dev.yml
3
+ # Please do not edit this file; any changes will be overwritten next time
4
+ # schema_dev gets run.
5
+ ---
6
+ sudo: false
7
+ rvm:
8
+ - 2.1.3
9
+ gemfile:
10
+ - gemfiles/activerecord-4.2.1/Gemfile.postgresql
11
+ - gemfiles/activerecord-4.2.1/Gemfile.sqlite3
12
+ env: POSTGRESQL_DB_USER=postgres
13
+ addons:
14
+ postgresql: '9.4'
15
+ before_script: bundle exec rake create_databases
16
+ after_script: bundle exec rake drop_databases
17
+ script: bundle exec rake travis
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 Nathan Griffith
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,121 @@
1
+ # Viewller
2
+
3
+ Viewller is a gem for creating and managing materialized views. Where possible, it uses the native
4
+ `MATERIALIZED VIEW` implementation, falling back to a standard table when no native implementation
5
+ exists. It integrates with ActiveRecord models, enabling easy data access.
6
+
7
+ Currently, Sqlite and PostgreSQL are supported, with other database engines on the way.
8
+
9
+ ## Installation
10
+
11
+ Add this line to your application's Gemfile (before executing `bundle`):
12
+
13
+ ```ruby
14
+ gem 'viewller'
15
+ ```
16
+
17
+ Optionally, to install the `viewller.rb` configuration initializer, execute:
18
+
19
+ $ rails g viewller:install
20
+
21
+ ## Usage
22
+
23
+ To insert and begin querying against a materialized view, include the `Viewller::MaterializedView`
24
+ module in your model, and then define the underlying query using a standard ActiveRecord query
25
+ object:
26
+
27
+ ```ruby
28
+ class PostsCommentsByUserTally < ActiveRecord::Base
29
+ include Viewller::MaterializedView
30
+
31
+ self.primary_key = :user_id
32
+
33
+ def self.query
34
+ User.eager_load(:posts, :comments)
35
+ .select('users.id AS user_id')
36
+ .select('COUNT(posts.id) AS posts_count')
37
+ .select('COUNT(comments.id) AS comments_count')
38
+ end
39
+ end
40
+ ```
41
+
42
+ The following commands can be used to create or drop the underlying views. These should generally
43
+ be run in a migration, but can be run outside of a migration by excluding the `self` argument.
44
+
45
+ ```ruby
46
+ PostsCommentsByUserTally.create_view(self)
47
+ PostsCommentsByUserTally.drop_view(self)
48
+ ```
49
+
50
+ To keep the view up-to-date, you'll need to call the following line in a recurring background job:
51
+
52
+ ```ruby
53
+ PostsCommentsByUserTally.refresh_view
54
+ ```
55
+
56
+ ## Concurrent Refresh
57
+
58
+ By default, Viewller will avoid holding a lock on the view/table during the duration of the
59
+ `refresh_view` command. On PostgreSQL, a native `CONCURRENT` refresh is used, while on other
60
+ databases the refresh is performed on a separate table and then synchronized at the ruby layer.
61
+
62
+ This can only be accomplished if a `primary_key` (or `primary_keys`) is specified on the model.
63
+ When such a key is present, the `create_view` command will also create a unique index on the
64
+ column (or set of columns). If no unique index is available, Viewller will fall back to a faster
65
+ (but locking) refresh.
66
+
67
+ To disable any part of these default behaviors, the following parameters are available:
68
+
69
+ ```ruby
70
+ # Skip creating the UNIQUE index
71
+ PostsCommentsByUserTally.create_view(self, skip_index: true)
72
+
73
+ # Don't refresh concurrently
74
+ PostsCommentsByUserTally.refresh_view(async: false)
75
+
76
+ # Disable these behaviors globally
77
+ Viewller.configure do |config|
78
+ config.auto_index_primary_key = false
79
+ config.refresh_concurrently = false
80
+ end
81
+ ```
82
+
83
+ ## Nested Views
84
+
85
+ Sometimes, materialized must refer to other materialized views. This can make it difficult
86
+ to create, refresh, or drop a set of views at once, in the correct order. Thankfully, Viewller
87
+ has you covered with the `depends_on` directive:
88
+
89
+ ```ruby
90
+ class CommentsPerPostByDayTally < ActiveRecord::Base
91
+ include Viewller::MaterializedView
92
+
93
+ self.primary_key = :date
94
+
95
+ depends_on [PostsByDateTally, CommentsByDateTally]
96
+
97
+ def self.query
98
+ posts, comments = PostsByDateTally.table_name, CommentsByDateTally.table_name
99
+ PostsByDateTally.joins("LEFT OUTER JOIN #{comments} ON #{posts}.date = #{comments}.date")
100
+ .select("SUM(#{comments}.value) / SUM(#{posts}.value) comments_per_post")
101
+ .select("#{posts}.date")
102
+ .group("#{posts}.date")
103
+ end
104
+ end
105
+ ```
106
+
107
+ To create, refresh, and drop the view and all of its dependent views, in the correct order,
108
+ simply add `dependents: true` to the standard methods:
109
+
110
+ ```ruby
111
+ CommentsPerPostByDayTally.create_view(self, dependents: true)
112
+ CommentsPerPostByDayTally.refresh_view(dependents: true)
113
+ CommentsPerPostByDayTally.drop_view(self, dependents: true)
114
+ ```
115
+
116
+ ## TODO
117
+
118
+ * Add support for triggered refreshes (both DB-level and ruby-level)
119
+ * Extract vendor-specific implementations into separate providers
120
+ * Implement more providers (MySQL, Oracle, MongoDB, MSFT SQL, etc)
121
+ * Explore implementing trigger-based data updates
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'schema_dev/tasks'
@@ -0,0 +1,14 @@
1
+ require 'rails/generators/active_record'
2
+ module Viewller
3
+ module Generators
4
+ class InstallGenerator < Rails::Generators::Base
5
+
6
+ desc 'Copies viewller config to your initializers.'
7
+ source_root File.expand_path('../templates', __FILE__)
8
+
9
+ def create_config_file
10
+ template 'config_initializer.rb', 'config/initializers/sojourn.rb'
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,19 @@
1
+ Viewller.configure do |config|
2
+
3
+ # By default, viewller uses native materialized view implementations when available, falling
4
+ # back to a standard table when not. The fallback behavior can be forced by uncommenting below:
5
+ #
6
+ # config.use_native = false
7
+
8
+ # If a primary key is specified on the model, `create_view` will automatically create a unique
9
+ # index for it, unless the following line is uncommented:
10
+ #
11
+ # config.auto_index_primary_key = false
12
+
13
+ # If a primary key and unique index exist on the view, `refresh_view` will attempt a concurrent
14
+ # (slower but non-blocking) refresh. Uncommenting the following line will disable this behavior
15
+ # and cause Viewller to use a faster (but blocking) refresh:
16
+ #
17
+ # config.refresh_concurrently = false
18
+
19
+ end
@@ -0,0 +1,25 @@
1
+ module Viewller
2
+ def self.configure(&block)
3
+ block.call(config)
4
+ end
5
+
6
+ def self.config
7
+ @config ||= Configuration.new
8
+ end
9
+
10
+ class Configuration
11
+ attr_accessor :use_native, :auto_index_primary_key, :refresh_concurrently
12
+
13
+ def initialize
14
+ set_defaults
15
+ end
16
+
17
+ private
18
+
19
+ def set_defaults
20
+ self.use_native = true
21
+ self.auto_index_primary_key = true
22
+ self.refresh_concurrently = true
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,53 @@
1
+ module Viewller
2
+ module MaterializedView
3
+ def self.included(base)
4
+ base.extend(ClassMethods)
5
+ end
6
+
7
+ def readonly?
8
+ true
9
+ end
10
+
11
+ module ClassMethods
12
+ def depends_on(*args)
13
+ @depends_on ||= [args].flatten
14
+ end
15
+
16
+ def refresh_view(m = connection, dependents: nil, async: true)
17
+ dependencies.each { |d| d.refresh_view(m, async: async) } if dependents
18
+ m.execute "REFRESH MATERIALIZED VIEW #{'CONCURRENTLY ' if async}#{table_name}"
19
+ rescue ActiveRecord::StatementInvalid => e
20
+ raise unless e.message.strip.start_with? 'PG::Error: ERROR: CONCURRENTLY cannot be used'
21
+ refresh_view(m, dependents: dependents, async: false)
22
+ end
23
+
24
+ def drop_view(m = connection, dependents: false)
25
+ m.execute "DROP MATERIALIZED VIEW IF EXISTS #{table_name}"
26
+ dependencies.reverse_each { |d| d.drop_view(m) } if dependents
27
+ end
28
+
29
+ def create_view(m = connection, dependents: false, skip_index: false)
30
+ dependencies.each { |d| d.create_view(m) } if dependents
31
+ m.execute "CREATE MATERIALIZED VIEW #{table_name} AS (#{query.to_sql})"
32
+ m.add_index table_name, unique_key, unique: true unless skip_index || !unique_key
33
+ end
34
+
35
+ protected
36
+
37
+ def unique_key
38
+ (primary_keys if respond_to?(:primary_keys)) || primary_key
39
+ end
40
+
41
+ def dependency_graph
42
+ return @dependency_graph if @dependency_graph
43
+ @dependency_graph = TsortableHash[{ self => depends_on }]
44
+ depends_on.each { |d| @dependency_graph.merge!(d.dependency_graph) }
45
+ @dependency_graph
46
+ end
47
+
48
+ def dependencies
49
+ @dependencies ||= dependency_graph.tsort - [self]
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,3 @@
1
+ module Viewller
2
+ VERSION = '0.0.1'
3
+ end
data/lib/viewller.rb ADDED
@@ -0,0 +1,6 @@
1
+ require_relative 'viewller/version'
2
+ require_relative 'viewller/configuration'
3
+ require_relative 'viewller/materialized_view'
4
+
5
+ module Viewller
6
+ end
data/schema_dev.yml ADDED
@@ -0,0 +1,7 @@
1
+ ruby:
2
+ - 2.1.3
3
+ activerecord:
4
+ - 4.2.1
5
+ db:
6
+ - sqlite3
7
+ - postgresql
@@ -0,0 +1,55 @@
1
+ require 'support/spec_helper'
2
+
3
+ VIEW_NAME = 'test'
4
+
5
+ class User < ActiveRecord::Base
6
+ has_many :posts
7
+ end
8
+
9
+ class Post < ActiveRecord::Base
10
+ belongs_to :user
11
+ end
12
+
13
+ describe Viewller::MaterializedView do
14
+
15
+ before(:each) do
16
+ ActiveRecord::Schema.define do
17
+ execute "DROP MATERIALIZED VIEW IF EXISTS #{VIEW_NAME}"
18
+ create_table :users, force: true do
19
+ end
20
+ create_table :posts, force: true do |t|
21
+ t.references :user
22
+ end
23
+ create_table :comments, force: true do |t|
24
+ t.references :post
25
+ t.references :user
26
+ end
27
+ end
28
+ end
29
+
30
+ subject do
31
+ Class.new(ActiveRecord::Base) do
32
+ include Viewller::MaterializedView
33
+ self.table_name = VIEW_NAME
34
+
35
+ def self.query
36
+ User.eager_load(:posts).group('users.id, posts.id')
37
+ .select('users.id AS user_id, COUNT(posts.id) AS posts_count')
38
+ end
39
+ end
40
+ end
41
+
42
+ it { should respond_to(:create_view) }
43
+ it { should respond_to(:refresh_view) }
44
+ it { should respond_to(:drop_view) }
45
+
46
+ context 'after create' do
47
+ let(:opts) { {} }
48
+
49
+ before { subject.create_view(opts) }
50
+ after { subject.drop_view }
51
+
52
+ its(:last) { should eq([]) }
53
+ end
54
+
55
+ end
@@ -0,0 +1,6 @@
1
+ require 'rspec'
2
+ require 'active_record'
3
+ require 'viewller'
4
+ require 'schema_dev/rspec'
5
+
6
+ SchemaDev::Rspec.setup
@@ -0,0 +1,51 @@
1
+ require 'viewller'
2
+
3
+ raise 'Missing gem version' unless /\A\d+\.\d+\.\d+\z/.match?(Viewller::VERSION)
4
+
5
+ config = Viewller.config
6
+ raise 'Unexpected configuration defaults' unless [
7
+ config.use_native, config.auto_index_primary_key, config.refresh_concurrently
8
+ ] == [true, true, true]
9
+
10
+ Viewller.configure { |options| options.refresh_concurrently = false }
11
+ raise 'Configuration block was not applied' unless Viewller.config.refresh_concurrently == false
12
+
13
+ class RecordingConnection
14
+ attr_reader :statements
15
+
16
+ def initialize
17
+ @statements = []
18
+ end
19
+
20
+ def execute(sql)
21
+ @statements << sql
22
+ end
23
+ end
24
+
25
+ model = Class.new do
26
+ include Viewller::MaterializedView
27
+
28
+ def self.table_name
29
+ 'demo_view'
30
+ end
31
+
32
+ def self.query
33
+ Struct.new(:to_sql).new('SELECT 1 AS id')
34
+ end
35
+ end
36
+
37
+ raise 'Materialized view should be read-only' unless model.new.readonly?
38
+
39
+ connection = RecordingConnection.new
40
+ model.create_view(connection, skip_index: true)
41
+ model.refresh_view(connection, async: false)
42
+ model.drop_view(connection)
43
+
44
+ expected = [
45
+ 'CREATE MATERIALIZED VIEW demo_view AS (SELECT 1 AS id)',
46
+ 'REFRESH MATERIALIZED VIEW demo_view',
47
+ 'DROP MATERIALIZED VIEW IF EXISTS demo_view'
48
+ ]
49
+ raise "Unexpected SQL: #{connection.statements.inspect}" unless connection.statements == expected
50
+
51
+ puts 'Viewller smoke test passed'
data/viewller.gemspec ADDED
@@ -0,0 +1,25 @@
1
+ lib = File.expand_path('../lib', __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require 'viewller/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'viewller'
7
+ spec.version = Viewller::VERSION
8
+ spec.authors = ['Smudge']
9
+ spec.email = ['nathan@ngriffith.com']
10
+ spec.summary = 'Create and manage materialized views with ActiveRecord'
11
+ spec.description = 'Utilities for creating, refreshing, and dropping materialized views in ActiveRecord.'
12
+ spec.homepage = 'https://gitlab.com/smudge/viewller'
13
+ spec.license = 'MIT'
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(/^bin/) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(/^(test|spec|features)/)
18
+ spec.require_paths = ['lib']
19
+
20
+ spec.add_development_dependency 'rake', '~> 13.0'
21
+ spec.add_development_dependency 'rspec'
22
+ spec.add_development_dependency 'schema_dev', '~> 3.5'
23
+
24
+ spec.add_dependency 'railties', '>= 3.2.1'
25
+ end
metadata ADDED
@@ -0,0 +1,128 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: viewller
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Smudge
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: rake
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ -
17
+ - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: "13.0"
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ -
25
+ - ~>
26
+ - !ruby/object:Gem::Version
27
+ version: "13.0"
28
+ - !ruby/object:Gem::Dependency
29
+ name: rspec
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ -
33
+ - ">="
34
+ - !ruby/object:Gem::Version
35
+ version: "0"
36
+ type: :development
37
+ prerelease: false
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ -
41
+ - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: "0"
44
+ - !ruby/object:Gem::Dependency
45
+ name: schema_dev
46
+ requirement: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ -
49
+ - ~>
50
+ - !ruby/object:Gem::Version
51
+ version: "3.5"
52
+ type: :development
53
+ prerelease: false
54
+ version_requirements: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ -
57
+ - ~>
58
+ - !ruby/object:Gem::Version
59
+ version: "3.5"
60
+ - !ruby/object:Gem::Dependency
61
+ name: railties
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ -
65
+ - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: 3.2.1
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ -
73
+ - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: 3.2.1
76
+ description: "Utilities for creating, refreshing, and dropping materialized views in ActiveRecord."
77
+ email:
78
+ - nathan@ngriffith.com
79
+ executables: []
80
+ extensions: []
81
+ extra_rdoc_files: []
82
+ files:
83
+ - .gitignore
84
+ - .gitlab-ci.yml
85
+ - .rubocop.yml
86
+ - .travis.yml
87
+ - Gemfile
88
+ - LICENSE
89
+ - README.md
90
+ - Rakefile
91
+ - lib/generators/viewller/install_generator.rb
92
+ - lib/generators/viewller/templates/config_initializer.rb
93
+ - lib/viewller.rb
94
+ - lib/viewller/configuration.rb
95
+ - lib/viewller/materialized_view.rb
96
+ - lib/viewller/version.rb
97
+ - schema_dev.yml
98
+ - spec/materialized_view_spec.rb
99
+ - spec/support/spec_helper.rb
100
+ - test/smoke_test.rb
101
+ - viewller.gemspec
102
+ homepage: "https://gitlab.com/smudge/viewller"
103
+ licenses:
104
+ - MIT
105
+ metadata: {}
106
+ rdoc_options: []
107
+ require_paths:
108
+ - lib
109
+ required_ruby_version: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ -
112
+ - ">="
113
+ - !ruby/object:Gem::Version
114
+ version: "0"
115
+ required_rubygems_version: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ -
118
+ - ">="
119
+ - !ruby/object:Gem::Version
120
+ version: "0"
121
+ requirements: []
122
+ rubygems_version: 4.1.0
123
+ specification_version: 4
124
+ summary: Create and manage materialized views with ActiveRecord
125
+ test_files:
126
+ - spec/materialized_view_spec.rb
127
+ - spec/support/spec_helper.rb
128
+ - test/smoke_test.rb