activerecord-clean-db-structure 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6ee2c62ab134b71d0ccce4e02928038c983c4403
-  data.tar.gz: 6264e28734c7ca2066fefbd2a6ef3dc68e68c1c1
+SHA256:
+  metadata.gz: 2e2fac86758bda878d5fea13096bc67af323ddffbb73e889c4cfc40bcb5a28c3
+  data.tar.gz: 7f425da0e19c813789a988326a07a69fe5d54e6e5987af9cfa1965fe09b1e27f
 SHA512:
-  metadata.gz: bf88f74415c68337f2b9f409a358085d4d71c64be4696a942b973a2d34c420024f9bde791f3edb120c1df26357d8c3d0f5d47ed94892437b63ef72b388ae5175
-  data.tar.gz: 1b086ae9152601942d7999ab0491c2d5d4e812e34208688f5bd8d8437530d74c868b0ca6a5639b237b2cb9cdd80c3caa1cd7871c601100b586ee748a86fe5def
+  metadata.gz: b87f9ddf4b6e9a8203463627c88164af61202fd054450dde7ff8316e10bf9849e05aa7077c72dac05aef27f1b0796d3f9c41538bb939f62b8958652c9664bc21
+  data.tar.gz: 591a5cb637d76600972fc5552e1124d9bfe6c46823622bfd03fac2e04facd3baa7f74949894f2e6649d808eb59c2912dca8a1a22d4b6bad2c429972d8b9a957c

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## Unreleased
+
+* ...
+
+## 0.4.0    2019-08-27
+
+* Add "indexes_after_tables" option to allow indexes to be placed following the respective tables [#13](https://github.com/lfittl/activerecord-clean-db-structure/pull/13) [Giovanni Kock Bonetti](https://github.com/giovannibonetti)
+* Add "order_schema_migrations_values" option to prevent schema_migrations values causing merge conflicts [#15](https://github.com/lfittl/activerecord-clean-db-structure/pull/15) [Nicke van Oorschot](https://github.com/nvanoorschot)
+* Add "order_column_definitions" option to sort table columns alphabetically [#11](https://github.com/lfittl/activerecord-clean-db-structure/pull/11) [RKushnir](https://github.com/RKushnir)
+* Generalize handling of schema names to not assume public
+* Rails 6 support
+  * Fix Rails 6 compatibility [#16](https://github.com/lfittl/activerecord-clean-db-structure/pull/16) [Giovanni Kock Bonetti](https://github.com/giovannibonetti)
+  * Fix handling of multiple structure.sql files
+* Remove Postgres 12 specific GUCs
+* Generalize handling of schema names to not assume public
+* Fix whitespace issue for config settings, remove default_with_oids
+
+
 ## 0.3.0    2019-05-07
 
 * Add "ignore_ids" option to allow disabling of primary key substitution logic [#12](https://github.com/lfittl/activerecord-clean-db-structure/pull/12) [Vladimir Dementyev](https://github.com/palkan)

data/Gemfile.lock

@@ -1,24 +1,23 @@
 PATH
   remote: .
   specs:
-    activerecord-clean-db-structure (0.3.0)
+    activerecord-clean-db-structure (0.4.0)
       activerecord (>= 4.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.3)
-      activesupport (= 5.2.3)
-    activerecord (5.2.3)
-      activemodel (= 5.2.3)
-      activesupport (= 5.2.3)
-      arel (>= 9.0)
-    activesupport (5.2.3)
+    activemodel (6.0.0)
+      activesupport (= 6.0.0)
+    activerecord (6.0.0)
+      activemodel (= 6.0.0)
+      activesupport (= 6.0.0)
+    activesupport (6.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
+      zeitwerk (~> 2.1, >= 2.1.8)
     concurrent-ruby (1.1.5)
     i18n (1.6.0)
       concurrent-ruby (~> 1.0)
@@ -27,6 +26,7 @@ GEM
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    zeitwerk (2.1.9)
 
 PLATFORMS
   ruby
@@ -36,4 +36,4 @@ DEPENDENCIES
   rake (~> 0)
 
 BUNDLED WITH
-   1.16.1
+   1.17.3

data/README.md

@@ -38,6 +38,55 @@ Rails.application.configure do
 end
 ```
 
+## Other options
+
+You can optionally have indexes following the respective tables setting `indexes_after_tables`:
+
+```ruby
+Rails.application.configure do
+  config.activerecord_clean_db_structure.indexes_after_tables = true
+end
+```
+
+When it is enabled the structure looks like this:
+
+```sql
+CREATE TABLE public.users (
+    id SERIAL PRIMARY KEY,
+    tenant_id integer,
+    email text NOT NULL
+);
+
+CREATE INDEX index_users_on_tentant_id ON public.users USING btree (tenant_id);
+CREATE UNIQUE INDEX index_users_on_email ON public.users USING btree (email);
+```
+
+To enable sorting the table column definitions alphabetically, discarding the actual order provided by `pg_dump`, set `order_column_definitions`:
+
+```ruby
+Rails.application.configure do
+  config.activerecord_clean_db_structure.order_column_definitions = true
+end
+```
+
+You can have the schema_migrations values reorganized to prevent merge conflicts by setting `order_schema_migrations_values`:
+
+```ruby
+Rails.application.configure do
+  config.activerecord_clean_db_structure.order_schema_migrations_values = true
+end
+```
+
+When it is enabled the values are ordered chronological and the semicolon is placed on a separate line:
+
+```sql
+INSERT INTO "schema_migrations" (version) VALUES
+ ('20190503120501')
+,('20190508123941')
+,('20190508132644')
+;
+```
+
 ## Authors
 
 * [Lukas Fittl](https://github.com/lfittl)

data/lib/activerecord-clean-db-structure/clean_dump.rb

@@ -15,8 +15,11 @@ module ActiveRecordCleanDbStructure
 
       # Remove version-specific output
       dump.gsub!(/^-- Dumped.*/, '')
-      dump.gsub!(/^SET row_security = off;$/, '') # 9.5
-      dump.gsub!(/^SET idle_in_transaction_session_timeout = 0;$/, '') # 9.6
+      dump.gsub!(/^SET row_security = off;\n/m, '') # 9.5
+      dump.gsub!(/^SET idle_in_transaction_session_timeout = 0;\n/m, '') # 9.6
+      dump.gsub!(/^SET default_with_oids = false;\n/m, '') # all older than 12
+      dump.gsub!(/^SET xmloption = content;\n/m, '') # 12
+      dump.gsub!(/^SET default_table_access_method = heap;\n/m, '') # 12
 
       # Remove pg_stat_statements extension (its not relevant to the code)
       dump.gsub!(/^CREATE EXTENSION IF NOT EXISTS pg_stat_statements.*/, '')
@@ -30,7 +33,7 @@ module ActiveRecordCleanDbStructure
       dump.gsub!(/^COMMENT ON EXTENSION .*/, '')
 
       # Remove useless, version-specific parts of comments
-      dump.gsub!(/^-- (.*); Schema: (public|-); Owner: -.*/, '-- \1')
+      dump.gsub!(/^-- (.*); Schema: ([\w_\.]+|-); Owner: -.*/, '-- \1')
 
       # Remove useless comment lines
       dump.gsub!(/^--$/, '')
@@ -41,8 +44,8 @@ module ActiveRecordCleanDbStructure
         # This is a bit optimistic, but works as long as you don't have an id field thats not a sequence/uuid
         dump.gsub!(/^    id integer NOT NULL(,)?$/, '    id SERIAL PRIMARY KEY\1')
         dump.gsub!(/^    id bigint NOT NULL(,)?$/, '    id BIGSERIAL PRIMARY KEY\1')
-        dump.gsub!(/^    id uuid DEFAULT (public\.)?uuid_generate_v4\(\) NOT NULL(,)?$/, '    id uuid DEFAULT \1uuid_generate_v4() PRIMARY KEY\2')
-        dump.gsub!(/^    id uuid DEFAULT (public\.)?gen_random_uuid\(\) NOT NULL(,)?$/, '    id uuid DEFAULT \1gen_random_uuid() PRIMARY KEY\2')
+        dump.gsub!(/^    id uuid DEFAULT ([\w_]+\.)?uuid_generate_v4\(\) NOT NULL(,)?$/, '    id uuid DEFAULT \1uuid_generate_v4() PRIMARY KEY\2')
+        dump.gsub!(/^    id uuid DEFAULT ([\w_]+\.)?gen_random_uuid\(\) NOT NULL(,)?$/, '    id uuid DEFAULT \1gen_random_uuid() PRIMARY KEY\2')
         dump.gsub!(/^CREATE SEQUENCE [\w\.]+_id_seq\s+(AS integer\s+)?START WITH 1\s+INCREMENT BY 1\s+NO MINVALUE\s+NO MAXVALUE\s+CACHE 1;$/, '')
         dump.gsub!(/^ALTER SEQUENCE [\w\.]+_id_seq OWNED BY .*;$/, '')
         dump.gsub!(/^ALTER TABLE ONLY [\w\.]+ ALTER COLUMN id SET DEFAULT nextval\('[\w\.]+_id_seq'::regclass\);$/, '')
@@ -57,9 +60,9 @@ module ActiveRecordCleanDbStructure
       inherited_tables = dump.scan(inherited_tables_regexp).map(&:first)
       dump.gsub!(inherited_tables_regexp, '')
       inherited_tables.each do |inherited_table|
-        dump.gsub!(/ALTER TABLE ONLY (public\.)?#{inherited_table}[^;]+;/, '')
+        dump.gsub!(/ALTER TABLE ONLY ([\w_]+\.)?#{inherited_table}[^;]+;/, '')
 
-        index_regexp = /CREATE INDEX ([\w_]+) ON (public\.)?#{inherited_table}[^;]+;/m
+        index_regexp = /CREATE INDEX ([\w_]+) ON ([\w_]+\.)?#{inherited_table}[^;]+;/m
         dump.scan(index_regexp).map(&:first).each do |inherited_table_index|
           dump.gsub!("-- Name: #{inherited_table_index}; Type: INDEX", '')
         end
@@ -67,31 +70,122 @@ module ActiveRecordCleanDbStructure
       end
 
       # Remove partitioned tables
-      partitioned_tables_regexp = /-- Name: ([\w_\.]+); Type: TABLE\n\n[^;]+?PARTITION OF [\w_\.]+\n[^;]+?;/m
-      partitioned_tables = dump.scan(partitioned_tables_regexp).map(&:first)
-      dump.gsub!(partitioned_tables_regexp, '')
+      partitioned_tables = []
+
+      # Postgres 12 pg_dump will output separate ATTACH PARTITION statements (even when run against an 11 or older server)
+      partitioned_tables_regexp1 = /ALTER TABLE ONLY [\w_\.]+ ATTACH PARTITION ([\w_\.]+)/
+      partitioned_tables += dump.scan(partitioned_tables_regexp1).map(&:last)
+
+      # Earlier versions use an inline PARTITION OF
+      partitioned_tables_regexp2 = /-- Name: ([\w_\.]+); Type: TABLE\n\n[^;]+?PARTITION OF [\w_\.]+\n[^;]+?;/m
+      partitioned_tables += dump.scan(partitioned_tables_regexp2).map(&:first)
+
       partitioned_tables.each do |partitioned_table|
-        dump.gsub!(/ALTER TABLE ONLY (public\.)?#{partitioned_table}[^;]+;/, '')
+        partitioned_schema_name, partitioned_table_name_only = partitioned_table.split('.', 2)
+        dump.gsub!(/-- Name: #{partitioned_table_name_only}; Type: TABLE/, '')
+        dump.gsub!(/CREATE TABLE #{partitioned_table} \([^;]+;/m, '')
+        dump.gsub!(/ALTER TABLE ONLY ([\w_\.]+) ATTACH PARTITION #{partitioned_table}[^;]+;/m, '')
+
+        dump.gsub!(/ALTER TABLE ONLY ([\w_]+\.)?#{partitioned_table}[^;]+;/, '')
         dump.gsub!(/-- Name: #{partitioned_table} [^;]+; Type: DEFAULT/, '')
 
-        index_regexp = /CREATE INDEX ([\w_]+) ON (public\.)?#{partitioned_table}[^;]+;/m
-        dump.scan(index_regexp).map(&:first).each do |partitioned_table_index|
+        index_regexp = /CREATE (UNIQUE )?INDEX ([\w_]+) ON ([\w_]+\.)?#{partitioned_table}[^;]+;/m
+        dump.scan(index_regexp).each do |m|
+          partitioned_table_index = m[1]
           dump.gsub!("-- Name: #{partitioned_table_index}; Type: INDEX ATTACH", '')
           dump.gsub!("-- Name: #{partitioned_table_index}; Type: INDEX", '')
-          dump.gsub!(/ALTER INDEX ([\w_\.]+) ATTACH PARTITION (public\.)?#{partitioned_table_index};/, '')
+          dump.gsub!(/ALTER INDEX ([\w_\.]+) ATTACH PARTITION ([\w_]+\.)?#{partitioned_table_index};/, '')
         end
         dump.gsub!(index_regexp, '')
 
-        dump.gsub!(/-- Name: #{partitioned_table}_pkey; Type: INDEX ATTACH\n\n[^;]+?ATTACH PARTITION (public\.)?#{partitioned_table}_pkey;/, '')
+        dump.gsub!(/-- Name: #{partitioned_table}_pkey; Type: INDEX ATTACH\n\n[^;]+?ATTACH PARTITION ([\w_]+\.)?#{partitioned_table}_pkey;/, '')
       end
       # This is mostly done to allow restoring Postgres 11 output on Postgres 10
       dump.gsub!(/CREATE INDEX ([\w_]+) ON ONLY/, 'CREATE INDEX \\1 ON')
 
-      # Remove whitespace between schema migration INSERTS to make editing easier
-      dump.gsub!(/^(INSERT INTO schema_migrations .*)\n\n/, "\\1\n")
+      if options[:order_schema_migrations_values] == true
+        schema_migrations_cleanup
+      else
+        # Remove whitespace between schema migration INSERTS to make editing easier
+        dump.gsub!(/^(INSERT INTO schema_migrations .*)\n\n/, "\\1\n")
+      end
+
+      if options[:indexes_after_tables] == true
+        # Extract indexes, remove comments and place them just after the respective tables
+        indexes =
+          dump
+            .scan(/^CREATE.+INDEX.+ON.+\n/)
+            .group_by { |line| line.scan(/\b\w+\.\w+\b/).first }
+            .transform_values(&:join)
+
+        dump.gsub!(/^CREATE( UNIQUE)? INDEX \w+ ON .+\n+/, '')
+        dump.gsub!(/^-- Name: \w+; Type: INDEX\n+/, '')
+        indexes.each do |table, indexes_for_table|
+          dump.gsub!(/^(CREATE TABLE #{table}\b(:?[^;\n]*\n)+\);\n)/) { $1 + "\n" + indexes_for_table }
+        end
+      end
 
       # Reduce 2+ lines of whitespace to one line of whitespace
       dump.gsub!(/\n{2,}/m, "\n\n")
+
+      if options[:order_column_definitions] == true
+        dump.replace(order_column_definitions(dump))
+      end
+    end
+
+    def order_column_definitions(source)
+      result = []
+
+      parse_column_name = ->(line) { line.match(/^    "?([^" ]+)/)[1] }
+      with_column_separator = ->(line) { line.sub(/,?\n$/, ",\n") }
+      without_column_separator = ->(line) { line.sub(/,\n$/, "\n") }
+
+      inside_table = false
+      columns = []
+
+      source.each_line do |source_line|
+        if source_line.start_with?("CREATE TABLE")
+          inside_table = true
+          columns = []
+          result << source_line
+        elsif source_line.start_with?(");")
+          if inside_table
+            inside_table = false
+            columns.sort_by!(&:first)
+
+            columns[0..-2].each do |_, line|
+              result << with_column_separator[line]
+            end
+
+            result << without_column_separator[columns.last[1]]
+          end
+
+          result << source_line
+        elsif inside_table
+          columns << [parse_column_name[source_line], source_line]
+        else
+          result << source_line
+        end
+      end
+
+      result.join
+    end
+
+    private
+
+    # Cleanup of schema_migrations values to prevent merge conflicts:
+    # - sorts all values chronological
+    # - places the comma's in front of each value (except for the first)
+    # - places the semicolon on a separate last line
+    def schema_migrations_cleanup
+      # Read all schema_migrations values from the dump.
+      values = dump.scan(/^(\(\'\d{14}\'\))[,;]\n/).flatten.sort
+
+      # Replace the schema_migrations values.
+      dump.sub!(
+        /(?<=INSERT INTO "schema_migrations" \(version\) VALUES).+;\n*/m,
+        "\n #{values.join("\n,")}\n;\n\n"
+      )
     end
   end
 end

data/lib/activerecord-clean-db-structure/tasks/clean_db_structure.rake

@@ -1,18 +1,23 @@
 require 'activerecord-clean-db-structure/clean_dump'
 
 Rake::Task['db:structure:dump'].enhance do
-  filenames = ENV['DB_STRUCTURE']
+  filenames = []
+  filenames << ENV['DB_STRUCTURE'] if ENV.key?('DB_STRUCTURE')
 
   if ActiveRecord::VERSION::MAJOR >= 6
-    ActiveRecord::Tasks::DatabaseTasks.for_each do |spec_name|
-      filenames ||= Rails.application.config.paths['db'].map do |path|
-        File.join(path, spec_name + '_structure.sql')
+    # Based on https://github.com/rails/rails/pull/36560/files
+    databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
+    ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |spec_name|
+      Rails.application.config.paths['db'].each do |path|
+        filenames << File.join(path, spec_name + '_structure.sql')
       end
     end
   end
 
-  filenames ||= Rails.application.config.paths['db'].map do |path|
-    File.join(path, 'structure.sql')
+  unless filenames.present?
+    Rails.application.config.paths['db'].each do |path|
+      filenames << File.join(path, 'structure.sql')
+    end
   end
 
   filenames.each do |filename|

data/lib/activerecord-clean-db-structure/version.rb

@@ -1,3 +1,3 @@
 module ActiveRecordCleanDbStructure
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-clean-db-structure
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Lukas Fittl
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-07 00:00:00.000000000 Z
+date: 2019-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -76,8 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Automatic cleanup for the Rails db/structure.sql file (ActiveRecord/PostgreSQL)