activerecord-pg-format-db-structure 0.1.0

data/README.md

@@ -0,0 +1,367 @@
+# activerecord-pg-format-db-structure
+
+Automatically cleans up your `structure.sql` file after each rails migration.
+
+By default, it will:
+
+* Inline primary key declarations
+* Inline SERIAL type declarations
+* Inline table constraints
+* Move index creation below their corresponding tables
+* Group `ALTER TABLE` statements into a single statement per table
+* Removes unnecessary whitespace
+
+The task will transform this raw `structure.sql`:
+
+<details>
+
+<summary>Click to expand</summary>
+
+```sql
+--
+-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
+
+
+--
+-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions';
+
+--
+-- Name: comments; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.comments (
+    id bigint NOT NULL,
+    user_id bigint NOT NULL,
+    post_id bigint NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: comments_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.comments_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: comments_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.comments_id_seq OWNED BY public.comments.id;
+
+--
+-- Name: posts; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.posts (
+    id bigint NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: posts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.posts_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: posts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.posts_id_seq OWNED BY public.posts.id;
+
+
+--
+-- Name: users; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.users (
+    id bigint NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.users_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id;
+
+--
+-- Name: comments id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.comments ALTER COLUMN id SET DEFAULT nextval('public.comments_id_seq'::regclass);
+
+--
+-- Name: posts id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.posts ALTER COLUMN id SET DEFAULT nextval('public.posts_id_seq'::regclass);
+
+--
+-- Name: users id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass);
+
+--
+-- Name: comments comments_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.comments
+    ADD CONSTRAINT comments_pkey PRIMARY KEY (id);
+
+--
+-- Name: posts posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.posts
+    ADD CONSTRAINT posts_pkey PRIMARY KEY (id);
+
+--
+-- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users
+    ADD CONSTRAINT users_pkey PRIMARY KEY (id);
+
+--
+-- Name: comments fk_rails_0000000001; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.comments
+    ADD CONSTRAINT fk_rails_0000000001 FOREIGN KEY (post_id) REFERENCES public.posts(id);
+
+--
+-- Name: comments fk_rails_0000000002; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.comments
+    ADD CONSTRAINT fk_rails_0000000002 FOREIGN KEY (user_id) REFERENCES public.users(id);
+
+INSERT INTO "schema_migrations" (version) VALUES
+('20250124155339');
+```
+</details>
+
+into this much more compact and normalized version:
+
+```sql
+-- Name: pgcrypto; Type: EXTENSION
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
+
+
+-- Name: comments; Type: TABLE;
+
+CREATE TABLE public.comments (
+    id bigserial PRIMARY KEY,
+    user_id bigint NOT NULL,
+    post_id bigint NOT NULL,
+    created_at timestamp(6) NOT NULL,
+    updated_at timestamp(6) NOT NULL
+);
+
+
+-- Name: posts; Type: TABLE;
+
+CREATE TABLE public.posts (
+    id bigserial PRIMARY KEY,
+    created_at timestamp(6) NOT NULL,
+    updated_at timestamp(6) NOT NULL
+);
+
+
+-- Name: users; Type: TABLE;
+
+CREATE TABLE public.users (
+    id bigserial PRIMARY KEY,
+    created_at timestamp(6) NOT NULL,
+    updated_at timestamp(6) NOT NULL
+);
+
+ALTER TABLE ONLY public.comments
+  ADD CONSTRAINT fk_rails_0000000001 FOREIGN KEY (post_id) REFERENCES public.posts (id),
+  ADD CONSTRAINT fk_rails_0000000002 FOREIGN KEY (user_id) REFERENCES public.users (id);
+
+INSERT INTO "schema_migrations" (version) VALUES
+('20250124155339');
+```
+
+which is a lot more compact, easier to read, and reduces the risk of
+getting random diffs between machines after each migration.
+
+Those transformations are made by manipulating the SQL AST directly
+using [pg_query](https://github.com/pganalyze/pg_query), and each
+transformation is opt-in and can be run independently.
+
+You can also add your own transforms (see below).
+
+
+## Installation
+
+Add the following to your Gemfile:
+
+```ruby
+gem 'activerecord-clean-db-structure'
+```
+
+## Usage
+
+### Rails
+
+Adding the gem to your dependencies this will automatically hook the library into your `rake db:migrate` task.
+
+If you want to configure which transforms to use, you can configure the library with the following:
+
+```ruby
+Rails.application.configure do
+  config.activerecord_pg_format_db_structure.preprocessors = [
+    ActiveRecordPgFormatDbStructure::Preprocessors::RemoveWhitespaces
+  ]
+
+  config.activerecord_pg_format_db_structure.transforms = [
+    ActiveRecordPgFormatDbStructure::Transforms::RemoveCommentsOnExtensions,
+    ActiveRecordPgFormatDbStructure::Transforms::InlinePrimaryKeys,
+    # ActiveRecordPgFormatDbStructure::Transforms::InlineForeignKeys,
+    ActiveRecordPgFormatDbStructure::Transforms::InlineSerials,
+    ActiveRecordPgFormatDbStructure::Transforms::InlineConstraints,
+    ActiveRecordPgFormatDbStructure::Transforms::MoveIndicesAfterCreateTable,
+    ActiveRecordPgFormatDbStructure::Transforms::GroupAlterTableStatements
+  ]
+
+  config.activerecord_pg_format_db_structure.deparser = ActiveRecordPgFormatDbStructure::Deparser
+end
+```
+
+### Use outside of Rails
+
+```ruby
+require "activerecord-pg-format-db-structure/formatter"
+
+structure = File.read("db/structure.sql")
+formatted = ActiveRecordPgFormatDbStructure::Formatter.new.format(structure)
+File.write("db/structure.sql", formatted)
+```
+
+## Preprocessors
+
+### RemoveWhitespaces
+
+Remove unnecessary comment, whitespase and empty lines.
+
+## Transformers
+
+### RemoveCommentsOnExtensions
+
+Remove COMMENT statement applied to extensions
+
+### InlinePrimaryKeys
+
+Inlines primary keys with the table declaration
+
+### InlineForeignKeys
+
+Inline foreign key constraints.
+
+Note: using this transform makes the structure file no longer
+loadable, since tables should be created before a foreign key
+can target it, so it is not included by default.
+
+### InlineSerials
+
+Inline SERIAL declaration inside table declaration.
+
+Note: the logic looks for statements of this shape:
+
+```sql
+ALTER TABLE ONLY ts.tn ALTER COLUMN c SET DEFAULT nextval('ts.tn_c_seq'::regclass);
+```
+
+It also assumes that the associated sequence has default settings. A
+later version could try to be more strict / validate that the
+sequence indeed has default settings.
+
+### InlineConstraints
+
+Inline non-foreign key constraints into table declaration
+
+### MoveIndicesAfterCreateTable
+
+Move indice declaration just below the table they index
+
+### GroupAlterTableStatements
+
+Group alter table statements into one operation per
+table.
+
+Should be run after other operations that inline alter statements.
+
+## Deparser
+
+As of today, this is a bare implemenation that works with the current combination of tranformers.
+
+As of now, it will only deparse `CREATE TABLE`, `CREATE INDEX` and
+`ALTER TABLE` statements. Other statements will be kept unchanged from
+the input SQL.
+
+In order to support all statements, we will need to find a solution to more cleanly format SQL queries, as deparsing a `CREATE VIEW` statement will result in a single unreadable line if relying on `pg_query`.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ReifyAB/activerecord-pg-format-db-structure. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/ReifyAB/activerecord-pg-format-db-structure/blob/main/CODE_OF_CONDUCT.md).
+
+## Credits
+
+Using the awesome [pg_query](https://github.com/pganalyze/pg_query) that provides a ruby interface to the native Postgres SQL parser.
+
+Inspired by the [activerecord-clean-db-structure](https://github.com/lfittl/activerecord-clean-db-structure) gem by [Lukas Fittl](https://github.com/lfittl). I wanted to achieved something like that, but using a proper SQL parser instead of search / replace using regexps.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the activerecord-pg-format-db-structure project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/ReifyAB/activerecord-pg-format-db-structure/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/activerecord-pg-format-db-structure/deparser.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "pg_query"
+
+module ActiveRecordPgFormatDbStructure
+  # Returns a list of SQL strings from a list of PgQuery::RawStmt.
+  class Deparser
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+    end
+
+    def deparse_raw_statement(raw_statement)
+      case raw_statement.to_h
+      in stmt: { create_stmt: _ }
+        deparse_create_stmt(raw_statement.stmt.create_stmt)
+      in stmt: { index_stmt: _ }
+        deparse_index_stmt(raw_statement.stmt.index_stmt)
+      in stmt: { alter_table_stmt: _ }
+        deparse_alter_table_stmt(raw_statement.stmt.alter_table_stmt)
+      else
+        keep_original_string(raw_statement)
+      end
+    end
+
+    private
+
+    def keep_original_string(raw_statement)
+      start = raw_statement.stmt_location || 0
+      stop = start + raw_statement.stmt_len
+      source[start..stop]
+    end
+
+    def deparse_index_stmt(index_stmt)
+      deparse_stmt(index_stmt)
+    end
+
+    def deparse_alter_table_stmt(alter_table_stmt)
+      "\n#{
+        deparse_stmt(alter_table_stmt)
+          .gsub(" ADD ", "\n  ADD ")
+          .gsub(" ALTER COLUMN ", "\n  ALTER COLUMN ")
+      }"
+    end
+
+    def deparse_stmt(stmt)
+      "\n#{PgQuery.deparse_stmt(stmt)};"
+    end
+
+    def deparse_create_stmt(create_stmt)
+      table_str = "\n\n\n-- Name: #{create_stmt.relation.relname}; Type: TABLE;\n\n"
+      table_str << PgQuery.deparse_stmt(
+        PgQuery::CreateStmt.new(
+          **create_stmt.to_h,
+          table_elts: []
+        )
+      )
+      table_str.gsub!(/\(\)\z/, "")
+      table_str << "("
+      table_str << create_stmt.table_elts.map do |elt|
+        "\n    #{deparse_table_elt(elt)}"
+      end.join(",")
+      table_str << "\n);"
+      table_str
+    end
+
+    def deparse_table_elt(elt)
+      PgQuery.deparse_stmt(
+        PgQuery::CreateStmt.new(
+          relation: PgQuery::RangeVar.new(relname: "tmp"), table_elts: [elt]
+        )
+      ).sub(/\ACREATE TABLE ONLY tmp \(/, "").sub(/\)\z/, "")
+    end
+  end
+end

data/lib/activerecord-pg-format-db-structure/formatter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "deparser"
+require_relative "../activerecord-pg-format-db-structure"
+
+module ActiveRecordPgFormatDbStructure
+  # Formats & normalizes in place the given SQL string
+  class Formatter
+    attr_reader :preprocessors, :transforms, :deparser
+
+    def initialize(
+      preprocessors: DEFAULT_PREPROCESSORS,
+      transforms: DEFAULT_TRANSFORMS,
+      deparser: DEFAULT_DEPARSER
+    )
+      @preprocessors = preprocessors
+      @transforms = transforms
+      @deparser = deparser
+    end
+
+    def format(source)
+      preprocessors.each do |preprocessor|
+        preprocessor.new(source).preprocess!
+      end
+
+      raw_statements = PgQuery.parse(source).tree.stmts
+
+      transforms.each do |transform|
+        transform.new(raw_statements).transform!
+      end
+
+      raw_statements.map do |raw_statement|
+        deparser.new(source).deparse_raw_statement(raw_statement)
+      end.compact.join
+    end
+  end
+end

data/lib/activerecord-pg-format-db-structure/preprocessors/remove_whitespaces.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ActiveRecordPgFormatDbStructure
+  module Preprocessors
+    # Remove whitespace and SQL comments from an SQL string
+    class RemoveWhitespaces
+      attr_reader :source
+
+      def initialize(source)
+        @source = source
+      end
+
+      def preprocess!
+        # Remove trailing whitespace
+        source.gsub!(/[ \t]+$/, "")
+        source.gsub!(/\A\n/, "")
+        source.gsub!(/\n\n\z/, "\n")
+
+        # Remove useless comment lines
+        source.gsub!(/^--\n/, "")
+
+        # Remove useless, version-specific parts of comments
+        source.gsub!(/^-- (.*); Schema: ([\w.]+|-); Owner: -.*/, '-- \1')
+      end
+    end
+  end
+end

data/lib/activerecord-pg-format-db-structure/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecordPgFormatDbStructure
+  # Setup for Rails
+  class Railtie < Rails::Railtie
+    config.activerecord_pg_format_db_structure = ActiveSupport::OrderedOptions.new
+    config.activerecord_pg_format_db_structure.preprocessors = DEFAULT_PREPROCESSORS.dup
+    config.activerecord_pg_format_db_structure.transforms = DEFAULT_TRANSFORMS.dup
+    config.activerecord_pg_format_db_structure.deparser = DEFAULT_DEPARSER
+
+    rake_tasks do
+      load "activerecord-pg-format-db-structure/tasks/clean_db_structure.rake"
+    end
+  end
+end

data/lib/activerecord-pg-format-db-structure/tasks/clean_db_structure.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "../formatter"
+
+Rake::Task["db:schema:dump"].enhance do
+  formatter = ActiveRecordPgFormatDbStructure::Formatter.new(
+    **Rails.application.config.activerecord_pg_format_db_structure
+  )
+
+  ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config|
+    filename = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, :sql)
+    next unless File.exist?(filename)
+
+    formatted = formatter.format(File.read(filename))
+    File.write(filename, formatted)
+  end
+end

data/lib/activerecord-pg-format-db-structure/transforms/group_alter_table_statements.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "pg_query"
+
+module ActiveRecordPgFormatDbStructure
+  module Transforms
+    # Group alter table statements into one operation per
+    # table. Should be run after other operations that inline alter statements.
+    class GroupAlterTableStatements
+      attr_reader :raw_statements
+
+      def initialize(raw_statements)
+        @raw_statements = raw_statements
+      end
+
+      def transform!
+        alter_groups = extract_alter_table_statements!
+
+        return if alter_groups.empty?
+
+        insert_index = raw_statements.each_with_index.map do |s, i|
+          # after all tables, materialized views and indices
+          i if s.stmt.to_h in { create_stmt: _ } | { create_table_as_stmt: _ } | { index_stmt: _ }
+        end.compact.last
+
+        sort_groups(alter_groups).each do |_, alters| # rubocop:disable Style/HashEachMethods
+          alter = sort_alters(alters).reduce do |altera, alterb|
+            altera.stmt.alter_table_stmt.cmds = altera.stmt.alter_table_stmt.cmds + alterb.stmt.alter_table_stmt.cmds
+            altera
+          end
+          raw_statements.insert(insert_index + 1, alter)
+        end
+      end
+
+      private
+
+      def sort_alters(alters)
+        alters.sort_by do |alter|
+          case alter.stmt.alter_table_stmt.to_h
+          in cmds: [{
+            alter_table_cmd: {
+              subtype: :AT_AddConstraint,
+              def: { constraint: {
+                contype: :CONSTR_FOREIGN,
+                fk_attrs: [{string: {sval: fk_attr}}],
+              }}
+            }
+          }]
+            [1, fk_attr]
+          else
+            [2, ""]
+          end
+        end
+      end
+
+      def sort_groups(groups)
+        groups.sort_by { |relation, _| [relation[:schemaname], relation[:relname]] }.reverse
+      end
+
+      def extract_alter_table_statements!
+        alter_groups = {}
+        raw_statements.delete_if do |s|
+          next unless s.stmt.to_h in alter_table_stmt: {
+            objtype: :OBJECT_TABLE,
+            relation: {
+              schemaname:,
+              relname:
+            }
+          }
+
+          relation = { schemaname:, relname: }
+          alter_groups[relation] ||= []
+          alter_groups[relation] << s
+          true
+        end
+        alter_groups
+      end
+    end
+  end
+end