better_structure_sql 0.1.0

data/lib/better_structure_sql/schema_versions.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module BetterStructureSql
+  # Schema version storage and retrieval operations
+  #
+  # Provides class methods for storing, querying, and managing
+  # schema versions with automatic retention cleanup.
+  module SchemaVersions
+    class << self
+      # Store current schema from file
+      def store_current(connection = ActiveRecord::Base.connection)
+        config = BetterStructureSql.configuration
+        pg_version = PgVersion.detect(connection)
+
+        # Determine format and output mode
+        format_type = deduce_format_type(config.output_path)
+        output_mode = detect_output_mode(config.output_path)
+
+        # Read content
+        content, zip_archive, file_count = read_schema_content(config.output_path, output_mode)
+
+        return nil unless content
+
+        store(
+          content: content,
+          zip_archive: zip_archive,
+          format_type: format_type,
+          output_mode: output_mode,
+          pg_version: pg_version,
+          file_count: file_count,
+          connection: connection
+        )
+      end
+
+      # Store schema version with explicit parameters
+      def store(content:, format_type:, pg_version:, **options)
+        connection = options.fetch(:connection, ActiveRecord::Base.connection)
+        output_mode = options.fetch(:output_mode, 'single_file')
+        zip_archive = options[:zip_archive]
+        file_count = options[:file_count]
+
+        ensure_table_exists!(connection)
+
+        version = SchemaVersion.create!(
+          content: content,
+          zip_archive: zip_archive,
+          format_type: format_type,
+          output_mode: output_mode,
+          pg_version: pg_version,
+          file_count: file_count,
+          created_at: Time.current
+        )
+
+        cleanup!(connection)
+
+        version
+      end
+
+      # Retrieval methods
+      def latest
+        return nil unless table_exists?
+
+        SchemaVersion.latest
+      end
+
+      # Returns all stored schema versions
+      #
+      # @return [Array<SchemaVersion>] All versions ordered by creation date (newest first)
+      def all_versions
+        return [] unless table_exists?
+
+        SchemaVersion.order(created_at: :desc).to_a
+      end
+
+      # Finds schema version by ID
+      #
+      # @param id [Integer] Version ID
+      # @return [SchemaVersion, nil] Found version or nil
+      def find(id)
+        return nil unless table_exists?
+
+        SchemaVersion.find_by(id: id)
+      end
+
+      # Returns total count of stored versions
+      #
+      # @return [Integer] Version count
+      def count
+        return 0 unless table_exists?
+
+        SchemaVersion.count
+      end
+
+      # Returns versions filtered by format type
+      #
+      # @param format_type [String] Format type ('sql' or 'rb')
+      # @return [Array<SchemaVersion>] Matching versions
+      def by_format(format_type)
+        return [] unless table_exists?
+
+        SchemaVersion.by_format(format_type).order(created_at: :desc).to_a
+      end
+
+      # Retention management
+      def cleanup!(_connection = ActiveRecord::Base.connection)
+        return 0 unless table_exists?
+
+        config = BetterStructureSql.configuration
+        limit = config.schema_versions_limit
+
+        # Skip cleanup if unlimited (0)
+        return 0 if limit.zero?
+
+        # Delete oldest versions beyond limit
+        total_count = SchemaVersion.count
+        return 0 if total_count <= limit
+
+        versions_to_delete = total_count - limit
+        oldest_versions = SchemaVersion.oldest_first.limit(versions_to_delete)
+
+        deleted_count = 0
+        oldest_versions.each do |version|
+          version.destroy
+          deleted_count += 1
+        end
+
+        deleted_count
+      end
+
+      private
+
+      def read_schema_content(output_path, output_mode)
+        case output_mode
+        when 'single_file'
+          full_path = Rails.root.join(output_path)
+          return [nil, nil, nil] unless File.exist?(full_path)
+
+          content = File.read(full_path)
+          [content, nil, nil]
+
+        when 'multi_file'
+          full_path = Rails.root.join(output_path)
+          return [nil, nil, nil] unless Dir.exist?(full_path)
+
+          # Read all files and combine into single content
+          content = read_multi_file_content(full_path)
+
+          # Create ZIP archive from directory
+          zip_archive = ZipGenerator.create_from_directory(full_path)
+
+          # Count files
+          file_count = Dir.glob("#{full_path}/**/*.sql").count
+
+          [content, zip_archive, file_count]
+        end
+      end
+
+      def read_multi_file_content(base_path)
+        content_parts = []
+
+        # Read header
+        header_path = base_path.join('_header.sql')
+        content_parts << File.read(header_path) if File.exist?(header_path)
+
+        # Read manifest and embed as SQL comment for later extraction
+        manifest_path = base_path.join('_manifest.json')
+        if File.exist?(manifest_path)
+          manifest_json = File.read(manifest_path)
+          content_parts << "-- MANIFEST_JSON_START\n-- #{manifest_json.gsub("\n", "\n-- ")}\n-- MANIFEST_JSON_END"
+        end
+
+        # Read numbered directories in order (01_ through 10_)
+        # Use pattern that works with Dir.glob
+        Dir.glob(File.join(base_path, '*_*')).select { |f| File.directory?(f) }.sort.each do |dir|
+          Dir.glob(File.join(dir, '*.sql')).sort.each do |file|
+            content_parts << File.read(file)
+          end
+        end
+
+        content_parts.join("\n\n")
+      end
+
+      def detect_output_mode(path)
+        # No extension → directory → multi_file
+        # Has extension (.sql, .rb) → file → single_file
+        File.extname(path.to_s).empty? ? 'multi_file' : 'single_file'
+      end
+
+      def deduce_format_type(path)
+        # Deduce format from file extension in output_path
+        if path.to_s.end_with?('.rb')
+          'rb'
+        else
+          'sql' # Default to SQL for .sql or any other extension
+        end
+      end
+
+      def ensure_table_exists!(_connection)
+        return if table_exists?
+
+        raise Error, "Schema versions table does not exist. Run migration first:\n  " \
+                     "rails generate better_structure_sql:migration\n  " \
+                     'rails db:migrate'
+      end
+
+      def table_exists?
+        ActiveRecord::Base.connection.table_exists?('better_structure_sql_schema_versions')
+      rescue ActiveRecord::NoDatabaseError
+        false
+      end
+    end
+  end
+end

data/lib/better_structure_sql/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module BetterStructureSql
+  VERSION = '0.1.0'
+end

data/lib/better_structure_sql/zip_generator.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'zip'
+require 'stringio'
+
+module BetterStructureSql
+  # Handles creating and extracting ZIP archives for multi-file schema dumps
+  class ZipGenerator
+    MAX_FILES_IN_ZIP = 300_000
+    MAX_UNCOMPRESSED_SIZE = 800.megabytes
+
+    class ZipError < StandardError; end
+
+    # Create ZIP from existing directory
+    # @param dir_path [String, Pathname] The directory path
+    # @return [String] ZIP file binary data
+    def self.create_from_directory(dir_path)
+      buffer = Zip::OutputStream.write_buffer do |zip|
+        Dir.glob("#{dir_path}/**/*").sort.each do |file_path|
+          next if File.directory?(file_path)
+
+          relative_path = file_path.sub("#{dir_path}/", '')
+          zip.put_next_entry(relative_path)
+          zip.write File.read(file_path)
+        end
+      end
+
+      buffer.string
+    end
+
+    # Create ZIP from file map (path => content hash)
+    # @param file_map [Hash<String, String>] Hash of relative paths to content
+    # @return [String] ZIP file binary data
+    def self.create_from_file_map(file_map)
+      buffer = Zip::OutputStream.write_buffer do |zip|
+        file_map.sort.each do |path, content|
+          zip.put_next_entry(path)
+          zip.write content
+        end
+      end
+
+      buffer.string
+    end
+
+    # Extract ZIP to directory
+    # @param zip_binary [String] ZIP file binary data
+    # @param target_dir [String, Pathname] Target directory path
+    def self.extract_to_directory(zip_binary, target_dir)
+      FileUtils.mkdir_p(target_dir)
+
+      Zip::File.open_buffer(StringIO.new(zip_binary)) do |zip_file|
+        zip_file.each do |entry|
+          # Security: rubyzip already prevents path traversal, but verify anyway
+          next if entry.name.include?('..')
+
+          path = File.join(target_dir, entry.name)
+          FileUtils.mkdir_p(File.dirname(path))
+          entry.extract(path) unless File.exist?(path)
+        end
+      end
+    end
+
+    # Validate ZIP safety (prevent ZIP bombs)
+    # @param zip_binary [String] ZIP file binary data
+    # @raise [ZipError] if ZIP exceeds safety limits
+    def self.validate_zip!(zip_binary)
+      file_count = 0
+      total_size = 0
+
+      Zip::File.open_buffer(StringIO.new(zip_binary)) do |zip_file|
+        zip_file.each do |entry|
+          file_count += 1
+          total_size += entry.size
+
+          raise ZipError, "Too many files in ZIP (max #{MAX_FILES_IN_ZIP})" if file_count > MAX_FILES_IN_ZIP
+          raise ZipError, "ZIP content too large (max #{MAX_UNCOMPRESSED_SIZE} bytes)" if total_size > MAX_UNCOMPRESSED_SIZE
+        end
+      end
+    end
+  end
+end

data/lib/better_structure_sql.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative 'better_structure_sql/version'
+require_relative 'better_structure_sql/adapters/base_adapter'
+require_relative 'better_structure_sql/adapters/postgresql_config'
+require_relative 'better_structure_sql/adapters/registry'
+require_relative 'better_structure_sql/configuration'
+require_relative 'better_structure_sql/dependency_resolver'
+require_relative 'better_structure_sql/introspection'
+require_relative 'better_structure_sql/formatter'
+require_relative 'better_structure_sql/generators/base'
+require_relative 'better_structure_sql/generators/extension_generator'
+require_relative 'better_structure_sql/generators/type_generator'
+require_relative 'better_structure_sql/generators/sequence_generator'
+require_relative 'better_structure_sql/generators/table_generator'
+require_relative 'better_structure_sql/generators/index_generator'
+require_relative 'better_structure_sql/generators/foreign_key_generator'
+require_relative 'better_structure_sql/generators/view_generator'
+require_relative 'better_structure_sql/generators/materialized_view_generator'
+require_relative 'better_structure_sql/generators/function_generator'
+require_relative 'better_structure_sql/generators/trigger_generator'
+require_relative 'better_structure_sql/generators/domain_generator'
+require_relative 'better_structure_sql/database_version'
+require_relative 'better_structure_sql/pg_version'
+require_relative 'better_structure_sql/schema_version'
+require_relative 'better_structure_sql/schema_versions'
+require_relative 'better_structure_sql/file_writer'
+require_relative 'better_structure_sql/manifest_generator'
+require_relative 'better_structure_sql/zip_generator'
+require_relative 'better_structure_sql/schema_loader'
+require_relative 'better_structure_sql/migration_patch'
+require_relative 'better_structure_sql/dumper'
+require_relative 'better_structure_sql/railtie' if defined?(Rails::Railtie)
+require_relative 'better_structure_sql/engine' if defined?(Rails::Engine)
+
+# BetterStructureSql - Clean PostgreSQL schema dumps for Rails applications
+#
+# Replaces noisy structure.sql files with deterministic, maintainable output
+# using pure Ruby database introspection. Supports PostgreSQL, MySQL, and SQLite.
+module BetterStructureSql
+  # Base error class for all BetterStructureSql errors
+  class Error < StandardError; end
+
+  class << self
+    attr_writer :configuration
+
+    # Returns the current configuration instance
+    #
+    # @return [Configuration] The configuration object
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    # Configures BetterStructureSql with a block
+    #
+    # @yield [Configuration] The configuration object
+    # @example
+    #   BetterStructureSql.configure do |config|
+    #     config.output_path = 'db/structure'
+    #     config.include_extensions = true
+    #   end
+    def configure
+      yield(configuration)
+    end
+
+    # Resets configuration to default values
+    #
+    # @return [Configuration] A new configuration instance
+    def reset_configuration
+      @configuration = Configuration.new
+    end
+
+    # Check if BetterStructureSql has been configured
+    #
+    # @return [Boolean] True if configuration has been set
+    def configured?
+      !@configuration.nil?
+    end
+  end
+end

data/lib/generators/better_structure_sql/install_generator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'rails/generators/base'
+
+module BetterStructureSql
+  module Generators
+    # Rails generator for installing BetterStructureSql
+    #
+    # Creates initializer and optionally generates migration for schema_versions table.
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('templates', __dir__)
+
+      desc 'Creates BetterStructureSql initializer and optionally generates migration'
+
+      class_option :skip_migration,
+                   type: :boolean,
+                   default: false,
+                   desc: 'Skip migration generation for schema versions table'
+
+      # Copies initializer template to config/initializers
+      #
+      # @return [void]
+      def copy_initializer
+        template 'better_structure_sql.rb', 'config/initializers/better_structure_sql.rb'
+      end
+
+      # Creates migration for schema_versions table
+      #
+      # @return [void]
+      def create_migration
+        return if options[:skip_migration]
+
+        generate 'better_structure_sql:migration'
+      end
+
+      # Displays README after installation
+      #
+      # @return [void]
+      def show_readme
+        readme 'README' if behavior == :invoke
+      end
+    end
+  end
+end

data/lib/generators/better_structure_sql/migration_generator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module BetterStructureSql
+  module Generators
+    # Rails generator for creating schema_versions table migration
+    class MigrationGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path('templates', __dir__)
+
+      desc 'Creates migration for BetterStructureSql schema versions table'
+
+      # Creates migration file for schema_versions table
+      #
+      # @return [void]
+      def create_migration_file
+        migration_template(
+          'migration.rb.erb',
+          'db/migrate/create_better_structure_sql_schema_versions.rb',
+          migration_version: migration_version
+        )
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/better_structure_sql/templates/README

@@ -0,0 +1,49 @@
+===============================================================================
+
+BetterStructureSql has been installed!
+
+Configuration file created at:
+  config/initializers/better_structure_sql.rb
+
+To generate a clean schema dump, run:
+  rails db:schema:dump_better
+
+To replace Rails' default schema dump task, set in the initializer:
+  config.replace_default_dump = true
+
+Then you can use the standard command:
+  rails db:schema:dump
+
+Phase 1 features available:
+  ✓ Tables with columns, primary keys, and constraints
+  ✓ Indexes (including unique and partial)
+  ✓ Foreign keys with CASCADE/RESTRICT/SET NULL
+  ✓ PostgreSQL extensions
+  ✓ Clean, deterministic output
+
+Phase 2 features available:
+  ✓ Schema version storage
+  ✓ Version retention management
+  ✓ PostgreSQL version tracking
+
+To enable schema versioning:
+  1. Enable in config/initializers/better_structure_sql.rb:
+       config.enable_schema_versions = true
+  2. Run the migration (if not already generated):
+       rails db:migrate
+  3. Store current schema:
+       rails db:schema:store
+  4. List versions:
+       rails db:schema:versions
+  5. Cleanup old versions:
+       rails db:schema:cleanup
+
+Coming in Phase 3:
+  - Views and materialized views
+  - Functions and triggers
+  - Advanced PostgreSQL features
+
+For more information, visit:
+  https://github.com/example/better_structure_sql
+
+===============================================================================

data/lib/generators/better_structure_sql/templates/add_metadata_migration.rb.erb

@@ -0,0 +1,25 @@
+class AddMetadataToBetterStructureSqlSchemaVersions < ActiveRecord::Migration<%= migration_version %>
+  def up
+    # Add new columns
+    add_column :better_structure_sql_schema_versions, :content_size, :bigint
+    add_column :better_structure_sql_schema_versions, :line_count, :integer
+
+    # Backfill existing records
+    BetterStructureSql::SchemaVersion.reset_column_information
+    BetterStructureSql::SchemaVersion.find_each do |version|
+      version.update_columns(
+        content_size: version.content.bytesize,
+        line_count: version.content.lines.count
+      )
+    end
+
+    # Make columns non-nullable
+    change_column_null :better_structure_sql_schema_versions, :content_size, false
+    change_column_null :better_structure_sql_schema_versions, :line_count, false
+  end
+
+  def down
+    remove_column :better_structure_sql_schema_versions, :line_count
+    remove_column :better_structure_sql_schema_versions, :content_size
+  end
+end

data/lib/generators/better_structure_sql/templates/better_structure_sql.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+BetterStructureSql.configure do |config|
+  # Output path for structure dump
+  # Single file: 'db/structure.sql'
+  # Multi-file: 'db/schema' (directory) - RECOMMENDED for large projects
+  #   Benefits: better git diffs, easier navigation, AI-friendly organization
+  # NOTE: BetterStructureSql only supports SQL format dumps (structure.sql).
+  #       Using 'db/schema.rb' will skip replacement of dump/load tasks (can still store versions).
+  config.output_path = Rails.root.join('db/structure.sql')
+
+  # Schema search path (PostgreSQL only)
+  config.search_path = 'public'
+
+  # Feature toggles - what to include in dumps
+  # Features auto-skip if not supported by your database
+  config.include_extensions = true          # PostgreSQL only
+  config.include_custom_types = true        # PostgreSQL (ENUM, composite), MySQL (ENUM/SET)
+  config.include_domains = true             # PostgreSQL only
+  config.include_sequences = true           # PostgreSQL only
+  config.include_functions = true           # PostgreSQL, MySQL (stored procedures)
+  config.include_triggers = true            # All databases
+  config.include_views = true               # All databases
+  config.include_materialized_views = true  # PostgreSQL only
+
+  # Output formatting
+  config.add_section_spacing = true # Add blank lines between sections
+  config.sort_tables = false        # Sort tables alphabetically (or use dependency order)
+
+  # Multi-file output settings (only used when output_path is a directory)
+  config.max_lines_per_file = 500 # Target lines per file (soft limit)
+  config.overflow_threshold = 1.1       # Allow files to be up to 10% larger to avoid tiny files
+  config.generate_manifest = true       # Generate _manifest.json with statistics
+
+  # Schema versioning (stores schema history in database for rollback/comparison)
+  config.enable_schema_versions = true  # Store versions in database
+  config.schema_versions_limit = 10     # Keep last 10 versions (0 = unlimited)
+
+  # Replace default Rails schema dump/load tasks (opt-in)
+  # When true, db:schema:dump and db:schema:load will use BetterStructureSql automatically
+  # This also automatically sets config.active_record.schema_format = :sql
+  # When false (default), use explicit tasks: db:schema:dump_better and db:schema:load_better
+  # NOTE: Only works with SQL format. Silently ignored if output_path ends with '.rb'
+  config.replace_default_dump = false
+  config.replace_default_load = false
+end

data/lib/generators/better_structure_sql/templates/migration.rb.erb

@@ -0,0 +1,26 @@
+class CreateBetterStructureSqlSchemaVersions < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :better_structure_sql_schema_versions do |t|
+      t.text :content, null: false
+      t.binary :zip_archive, null: true
+      t.string :pg_version, null: false
+      t.string :format_type, null: false
+      t.string :output_mode, null: false
+      t.bigint :content_size, null: false
+      t.integer :line_count, null: false
+      t.integer :file_count, null: true
+      t.timestamp :created_at, null: false
+    end
+
+    add_index :better_structure_sql_schema_versions, :created_at, order: { created_at: :desc }
+    add_index :better_structure_sql_schema_versions, :output_mode
+
+    add_check_constraint :better_structure_sql_schema_versions,
+                         "format_type IN ('sql', 'rb')",
+                         name: 'format_type_check'
+
+    add_check_constraint :better_structure_sql_schema_versions,
+                         "output_mode IN ('single_file', 'multi_file')",
+                         name: 'output_mode_check'
+  end
+end