activerecord_dumper 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2257a54f22f47577b964bcc2ccc807350eae41e165cf450cfdaae99df2767770
+  data.tar.gz: 2865477d775d608804fa9c8fc3ae1c2a5a85920b3f62fdcad3799aec527f3d1c
+SHA512:
+  metadata.gz: 5ad675615728f99ddac5801b8456cda901e3be335867a64f3b537ce4e6070a5ea96cdec10d22294b945b3bddf04a55dcaa386454dc2a3f6cf15d814f3085e18e
+  data.tar.gz: 37d1aaa78937bfaadd31bab80634547274e6ce5734a95e5d0a3ae965d726bf3a9951142afeb5f9c59b49c40e077a566c67d09a7040a4aca7b5e8acd00429dbca

data/README.md

@@ -0,0 +1,68 @@
+# ActiveRecord Dumper
+[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/spijet/activerecord_dumper/blob/master/LICENSE)
+[![GitHub issues](https://img.shields.io/github/issues/spijet/activerecord_dumper.svg)](https://github.com/spijet/activerecord_dumper/issues)
+[![Build Status](https://img.shields.io/travis/spijet/activerecord_dumper.svg)](https://travis-ci.com/spijet/activerecord_dumper)
+[![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/spijet/activerecord_dumper/rspec.yml?label=RSpec)](https://github.com/spijet/activerecord_dumper/actions/workflows/rspec.yml)
+[![Coverage Status](https://img.shields.io/coveralls/spijet/activerecord_dumper.svg)](https://coveralls.io/github/spijet/activerecord_dumper?branch=master)
+
+ActiveRecordDumper is a fork of YamlDb gem without any explicit Rails dependencies. This
+way it can be used by any AR-enabled app (e.g. Sinatra) without pulling whole
+Rails in.
+
+YamlDB/ActiveRecordDumper is a database-independent format for dumping and
+restoring data. It complements the database-independent schema format found in
+`db/schema.rb`. The data is saved into `db/data.yml`. This can be used as a
+replacement for `mysqldump` or `pg_dump`, but it only supports features found in
+ActiveRecord-based (Rails, etc.) apps. Users, permissions, schemas, triggers,
+and other advanced database features are not supported by design.
+
+Any database that has an ActiveRecord adapter should work.
+This gem aims to support ActiveRecord versions 4.2 through 8.1.
+
+
+## Installation
+
+Simply add to your `Gemfile`:
+
+    gem 'activerecord_dumper'
+
+All rake tasks will then be available to you.
+
+## Usage (Rails)
+
+    rake db:data:dump ->   Dump contents of Rails database to db/data.yml
+    rake db:data:load ->   Load contents of db/data.yml into the database
+
+Further, there are tasks `db:dump` and `db:load` which do the entire database
+(the equivalent of running `db:schema:dump` followed by `db:data:load`). Also,
+there are other tasks recently added that allow the export of the database
+contents to/from multiple files (each one named after the table being dumped or
+loaded).
+
+    rake db:data:dump_dir ->   Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)
+    rake db:data:load_dir ->   Load contents of db/#{dir} into database (where dir is ENV['dir'] || 'base')
+
+In addition, we have plugins whereby you can export your database to/from
+various formats. We only deal with YAML and CSV right now, but you can easily
+write tools for your own formats (such as Excel or XML). To use another format,
+just load setting the "class" parameter to the class you are using. This
+defaults to "YamlDb::Helper" which is a refactoring of the old yaml_db code.
+We'll shorten this to use class nicknames in a little bit.
+
+## Examples
+
+One common use would be to switch your data from one database backend to
+another. For example, let's say you wanted to switch from SQLite to MySQL. You
+might execute the following steps:
+
+1. `rake db:dump`;
+
+2. Edit `config/database.yml` and change your adapter to `mysql`, set up database params;
+
+3. `mysqladmin create [database name]`
+
+4. `rake db:load`
+
+## Credits
+
+Created by Orion Henry and Adam Wiggins. Major updates by Ricardo Chimal Jr. and Nate Kidwell. Adapted for non-Rails usage by Serge Tkatchouk.

data/lib/activerecord_dumper/csv_db.rb

@@ -0,0 +1,73 @@
+module ActiveRecordDumper
+  module CSVDB
+    module Helper
+      def self.loader
+        Load
+      end
+
+      def self.dumper
+        Dump
+      end
+
+      def self.extension
+        'csv'
+      end
+    end
+
+    class Load < SerializationHelper::Load
+      def self.load_documents(io, truncate = true)
+        tables = {}
+        curr_table = nil
+        io.each do |line|
+          if /BEGIN_CSV_TABLE_DECLARATION(.+)END_CSV_TABLE_DECLARATION/ =~ line
+            curr_table = Regexp.last_match[1] if Regexp.last_match
+            tables[curr_table] = {}
+          elsif tables[curr_table]['columns']
+            tables[curr_table]['records'] << FasterCSV.parse(line)[0]
+          else
+            tables[curr_table]['columns'] = FasterCSV.parse(line)[0]
+            tables[curr_table]['records'] = []
+          end
+        end
+
+        tables.each_pair do |table_name, contents|
+          load_table(table_name, contents, truncate)
+        end
+      end
+    end
+
+    class Dump < SerializationHelper::Dump
+      def self.before_table(io, table)
+        io.write "BEGIN_CSV_TABLE_DECLARATION#{table}END_CSV_TABLE_DECLARATION\n"
+      end
+
+      def self.dump(io)
+        tables.each do |table|
+          before_table(io, table)
+          dump_table(io, table)
+          after_table(io, table)
+        end
+      end
+
+      def self.after_table(io, _table)
+        io.write ''
+      end
+
+      def self.dump_table_columns(io, table)
+        io.write(table_column_names(table).to_csv)
+      end
+
+      def self.dump_table_records(io, table)
+        column_names = table_column_names(table)
+        helper = SerializationHelper::Utils
+
+        each_table_page(table) do |records|
+          rows = helper.unhash_records(records, column_names)
+          records.each do |record|
+            io.write(record.to_csv)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/activerecord_dumper/rake_tasks.rb

@@ -0,0 +1,55 @@
+module ActiveRecordDumper
+  module RakeTasks
+    def self.data_dump_task
+      SerializationHelper::Base.new(helper).dump(db_dump_data_file(helper.extension))
+    end
+
+    def self.data_dump_dir_task
+      dir = ENV['dir'] || default_dir_name
+      SerializationHelper::Base.new(helper).dump_to_dir(dump_dir("/#{dir}"))
+    end
+
+    def self.data_load_task
+      SerializationHelper::Base.new(helper).load(db_dump_data_file(helper.extension))
+    end
+
+    def self.data_load_dir_task
+      dir = ENV['dir'] || 'base'
+      SerializationHelper::Base.new(helper).load_from_dir(dump_dir("/#{dir}"))
+    end
+
+    # Private class methods below.
+
+    def self.default_dir_name
+      Time.now.strftime('%FT%H%M%S')
+    end
+
+    def self.db_dump_data_file(extension = 'yml')
+      "#{dump_dir}/data.#{extension}"
+    end
+
+    def self.dump_dir(dir = '')
+      "#{root_dir}/db#{dir}"
+    end
+
+    def self.root_dir
+      return Rails.root if defined? Rails
+      return APP_DIR if defined? APP_DIR
+      return APP_ROOT if defined? APP_ROOT
+
+      if ENV.key? 'APP_DIR'
+        ENV['APP_DIR']
+      elsif ENV.key? 'APP_ROOT'
+        ENV['APP_ROOT']
+      end
+    end
+
+    def self.helper
+      format_class = ENV['class'] || 'ActiveRecordDumper::Helper'
+      format_class.constantize
+    end
+
+    private_class_method :default_dir_name, :db_dump_data_file,
+                         :dump_dir, :root_dir, :helper
+  end
+end

data/lib/activerecord_dumper/serialization_helper.rb

@@ -0,0 +1,212 @@
+require 'active_support/core_ext/kernel/reporting'
+
+module ActiveRecordDumper
+  module SerializationHelper
+    class Base
+      attr_reader :extension
+
+      def initialize(helper)
+        @dumper    = helper.dumper
+        @loader    = helper.loader
+        @extension = helper.extension
+      end
+
+      def dump(filename)
+        disable_logger
+        File.open(filename, 'w') do |file|
+          @dumper.dump(file)
+        end
+        reenable_logger
+      end
+
+      def dump_to_dir(dirname)
+        Dir.mkdir(dirname)
+        tables = @dumper.tables
+        tables.each do |table|
+          File.open("#{dirname}/#{table}.#{@extension}", 'w') do |io|
+            @dumper.before_table(io, table)
+            @dumper.dump_table io, table
+            @dumper.after_table(io, table)
+          end
+        end
+      end
+
+      def load(filename, truncate = true)
+        disable_logger
+        @loader.load(File.new(filename, 'r'), truncate)
+        reenable_logger
+      end
+
+      def load_from_dir(dirname, truncate = true)
+        Dir.entries(dirname).each do |filename|
+          next if /^\./ =~ filename
+
+          @loader.load(File.new("#{dirname}/#{filename}", 'r'), truncate)
+        end
+      end
+
+      def disable_logger
+        @@old_logger = ActiveRecord::Base.logger
+        ActiveRecord::Base.logger = nil
+      end
+
+      def reenable_logger
+        ActiveRecord::Base.logger = @@old_logger
+      end
+    end
+
+    class Load
+      def self.load(io, truncate = true)
+        ActiveRecord::Base.connection.transaction do
+          load_documents(io, truncate)
+        end
+      end
+
+      def self.truncate_table(table)
+        ActiveRecord::Base.connection.execute("TRUNCATE #{Utils.quote_table(table)}")
+      rescue Exception
+        ActiveRecord::Base.connection.execute("DELETE FROM #{Utils.quote_table(table)}")
+      end
+
+      def self.load_table(table, data, truncate = true)
+        column_names = data['columns']
+        truncate_table(table) if truncate
+        load_records(table, column_names, data['records'])
+        reset_pk_sequence!(table)
+      end
+
+      def self.load_records(table, column_names, records)
+        return if column_names.nil?
+
+        quoted_column_names = column_names.map do |column|
+          ActiveRecord::Base.connection.quote_column_name(column)
+        end.join(',')
+        quoted_table_name = Utils.quote_table(table)
+        records.each do |record|
+          quoted_values = record.map do |c|
+            ActiveRecord::Base.connection.quote(c)
+          end.join(',')
+          ActiveRecord::Base.connection.execute(
+            "INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})"
+          )
+        end
+      end
+
+      def self.reset_pk_sequence!(table_name)
+        if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
+          ActiveRecord::Base.connection.reset_pk_sequence!(table_name)
+        end
+      end
+    end
+
+    module Utils
+      def self.unhash(hash, keys)
+        keys.map { |key| hash[key] }
+      end
+
+      def self.unhash_records(records, keys)
+        records.each_with_index do |record, index|
+          records[index] = unhash(record, keys)
+        end
+
+        records
+      end
+
+      def self.convert_booleans(records, columns)
+        records.each do |record|
+          columns.each do |column|
+            next if is_boolean(record[column])
+
+            record[column] = convert_boolean(record[column])
+          end
+        end
+        records
+      end
+
+      def self.convert_boolean(value)
+        ['t', '1', true, 1].include?(value)
+      end
+
+      def self.boolean_columns(table)
+        columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
+        columns.map(&:name)
+      end
+
+      def self.is_boolean(value)
+        value.is_a?(TrueClass) || value.is_a?(FalseClass)
+      end
+
+      def self.quote_table(table)
+        ActiveRecord::Base.connection.quote_table_name(table)
+      end
+
+      def self.quote_column(column)
+        ActiveRecord::Base.connection.quote_column_name(column)
+      end
+    end
+
+    class Dump
+      def self.before_table(io, table); end
+
+      def self.dump(io)
+        tables.each do |table|
+          before_table(io, table)
+          dump_table(io, table)
+          after_table(io, table)
+        end
+      end
+
+      def self.after_table(io, table); end
+
+      def self.tables
+        ActiveRecord::Base.connection.tables.reject do |table|
+          %w[schema_info schema_migrations].include?(table)
+        end.sort
+      end
+
+      def self.dump_table(io, table)
+        return if table_record_count(table).zero?
+
+        dump_table_columns(io, table)
+        dump_table_records(io, table)
+      end
+
+      def self.table_column_names(table)
+        ActiveRecord::Base.connection.columns(table).map(&:name)
+      end
+
+      def self.each_table_page(table, records_per_page = 1000)
+        total_count = table_record_count(table)
+        pages = (total_count.to_f / records_per_page).ceil - 1
+        keys = sort_keys(table)
+        boolean_columns = Utils.boolean_columns(table)
+
+        (0..pages).to_a.each do |page|
+          query = Arel::Table.new(table).order(*keys).skip(records_per_page * page).take(records_per_page).project(Arel.sql('*'))
+          records = ActiveRecord::Base.connection.select_all(query.to_sql)
+          records = Utils.convert_booleans(records, boolean_columns)
+          yield records
+        end
+      end
+
+      def self.table_record_count(table)
+        ActiveRecord::Base.connection.select_one(
+          "SELECT COUNT(*) FROM #{Utils.quote_table(table)}"
+        ).values.first.to_i
+      end
+
+      # Return the first column as sort key unless the table looks like a
+      # standard has_and_belongs_to_many join table,
+      # in which case add the second "ID column"
+      def self.sort_keys(table)
+        first_column, second_column = table_column_names(table)
+
+        if [first_column, second_column].all? { |name| name =~ /_id$/ }
+          [Utils.quote_column(first_column), Utils.quote_column(second_column)]
+        else
+          [Utils.quote_column(first_column)]
+        end
+      end
+    end
+  end
+end

data/lib/activerecord_dumper/version.rb

@@ -0,0 +1,3 @@
+module ActiveRecordDumper
+  VERSION = '0.9.0'.freeze
+end

data/lib/activerecord_dumper.rb

@@ -0,0 +1,75 @@
+require 'rubygems'
+require 'yaml'
+require 'active_record'
+require 'activerecord_dumper/rake_tasks'
+require 'activerecord_dumper/version'
+require 'activerecord_dumper/serialization_helper'
+
+module ActiveRecordDumper
+  module Helper
+    def self.loader
+      Load
+    end
+
+    def self.dumper
+      Dump
+    end
+
+    def self.extension
+      'yml'
+    end
+  end
+
+  module Utils
+    def self.chunk_records(records)
+      yaml = [records].to_yaml
+      yaml.sub!(/---\s\n|---\n/, '')
+      yaml.sub!('- - -', '  - -')
+      yaml
+    end
+  end
+
+  class Dump < SerializationHelper::Dump
+    def self.dump_table_columns(io, table)
+      io.write("\n")
+      io.write({ table => { 'columns' => table_column_names(table) } }.to_yaml)
+    end
+
+    def self.dump_table_records(io, table)
+      table_record_header(io)
+
+      column_names = table_column_names(table)
+
+      each_table_page(table) do |records|
+        rows = SerializationHelper::Utils.unhash_records(
+          records.to_a, column_names
+        )
+        io.write(Utils.chunk_records(rows))
+      end
+    end
+
+    def self.table_record_header(io)
+      io.write("  records:\n")
+    end
+  end
+
+  class Load < SerializationHelper::Load
+    def self.load_documents(io, truncate = true)
+      YAML.load_stream(io) do |ydoc|
+        ydoc.keys.each do |table_name|
+          next if ydoc[table_name].nil?
+
+          load_table(table_name, ydoc[table_name], truncate)
+        end
+      end
+    end
+  end
+
+  if defined?(Rails::Railtie)
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load File.expand_path('tasks/activerecord_dumper_tasks.rake', __dir__)
+      end
+    end
+  end
+end

data/lib/tasks/activerecord_dumper_tasks.rake

@@ -0,0 +1,29 @@
+namespace :db do
+  desc 'Dump schema and data to db/schema.rb and db/data.yml'
+  task(dump: ['db:schema:dump', 'db:data:dump'])
+
+  desc 'Load schema and data from db/schema.rb and db/data.yml'
+  task(load: ['db:schema:load', 'db:data:load'])
+
+  namespace :data do
+    desc 'Dump contents of database to db/data.extension (defaults to yaml)'
+    task dump: :environment do
+      ActiveRecordDumper::RakeTasks.data_dump_task
+    end
+
+    desc 'Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)'
+    task dump_dir: :environment do
+      ActiveRecordDumper::RakeTasks.data_dump_dir_task
+    end
+
+    desc 'Load contents of db/data.extension (defaults to yaml) into database'
+    task load: :environment do
+      ActiveRecordDumper::RakeTasks.data_load_task
+    end
+
+    desc 'Load contents of db/data_dir into database'
+    task load_dir: :environment do
+      ActiveRecordDumper::RakeTasks.data_load_dir_task
+    end
+  end
+end

data/spec/activerecord_dumper/dump_spec.rb

@@ -0,0 +1,54 @@
+module ActiveRecordDumper
+  RSpec.describe Dump do
+    before do
+      allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection').as_null_object)
+      allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[mytable schema_info schema_migrations])
+      allow(ActiveRecord::Base.connection).to receive(:columns).with('mytable').and_return([double('a', name: 'a', type: :string), double('b', name: 'b', type: :string)])
+      allow(ActiveRecord::Base.connection).to receive(:select_one).and_return('count' => '2')
+      allow(ActiveRecord::Base.connection).to receive(:select_all).and_return([{ 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 }])
+      allow(Utils).to receive(:quote_table).with('mytable').and_return('mytable')
+    end
+
+    before(:each) do
+      allow(File).to receive(:open).with('dump.yml', 'w').and_yield(StringIO.new)
+      @io = StringIO.new
+    end
+
+    it 'returns a formatted string' do
+      Dump.table_record_header(@io)
+      @io.rewind
+      expect(@io.read).to eq("  records:\n")
+    end
+
+    it 'returns a yaml string that contains a table header and column names' do
+      allow(Dump).to receive(:table_column_names).with('mytable').and_return(%w[a b])
+      Dump.dump_table_columns(@io, 'mytable')
+      @io.rewind
+      expected_yml = <<EOYAML
+
+---
+mytable:
+  columns:
+  - a
+  - b
+EOYAML
+      expect(@io.read).to eq(expected_yml)
+    end
+
+    it 'dumps the records for a table in yaml to a given io stream' do
+      allow(SerializationHelper::Dump).to receive(:each_table_page).with('mytable').and_yield(
+        [{ 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 }]
+      )
+      Dump.dump_table_records(@io, 'mytable')
+      @io.rewind
+      expected_yml = <<EOYAML
+  records:
+  - - 1
+    - 2
+  - - 3
+    - 4
+EOYAML
+      expect(@io.read).to eq(expected_yml)
+    end
+  end
+end

data/spec/activerecord_dumper/integration_spec.rb

@@ -0,0 +1,49 @@
+require 'active_record/railtie'
+# require 'active_record'
+
+# connect to in-memory SQLite database
+ActiveRecord::Base.establish_connection(
+  adapter: 'sqlite3',
+  database: ':memory:'
+)
+
+# define a dummy User model
+class User < ActiveRecord::Base
+end
+
+# create the users table
+ActiveRecord::Schema.define do
+  self.verbose = false
+
+  create_table :users, force: true do |t|
+    t.string :username
+  end
+end
+
+# add some users
+User.create(
+  [
+    { username: 'alice' },
+    { username: 'bob' }
+  ]
+)
+
+RSpec.describe 'with real ActiveRecord,' do
+  it 'contains two users' do
+    expect(User.count).to eq(2)
+  end
+
+  it 'dumps the user records' do
+    @io = StringIO.new
+    ActiveRecordDumper::Dump.dump_table_records(@io, 'users')
+    @io.rewind
+    expected_yml = <<EOYAML
+  records:
+  - - 1
+    - alice
+  - - 2
+    - bob
+EOYAML
+    expect(@io.read).to eq(expected_yml)
+  end
+end

data/spec/activerecord_dumper/load_spec.rb

@@ -0,0 +1,42 @@
+module ActiveRecordDumper
+  RSpec.describe Load do
+    before do
+      allow(SerializationHelper::Utils).to(
+        receive(:quote_table).with('mytable').and_return('mytable')
+      )
+      allow(ActiveRecord::Base).to(
+        receive(:connection).and_return(double('connection').as_null_object)
+      )
+      allow(ActiveRecord::Base.connection).to receive(:transaction).and_yield
+    end
+
+    before(:each) do
+      @io = StringIO.new
+    end
+
+    it 'calls load structure for each document in the file' do
+      stream_struct = {
+        'mytable' => {
+          'columns' => %w[a b],
+          'records' => [[1, 2], [3, 4]]
+        }
+      }
+      expect(YAML).to receive(:load_stream).with(@io).and_yield(stream_struct)
+      table_struct = [
+        'mytable',
+        { 'columns' => %w[a b], 'records' => [[1, 2], [3, 4]] },
+        true
+      ]
+      expect(Load).to receive(:load_table).with(*table_struct)
+      Load.load(@io)
+    end
+
+    it 'calls load structure when the document contains no records' do
+      expect(YAML).to receive(:load_stream).with(@io).and_yield(
+        'mytable' => nil
+      )
+      expect(Load).not_to receive(:load_table)
+      Load.load(@io)
+    end
+  end
+end