yaml_db_with_schema_tables 0.2.4 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eeec6633e75b68276003927b191f5e4217c2e2b0
-  data.tar.gz: 577d9281469b47494ebc918566a75a1809a334dc
+  metadata.gz: 56086fd52fca521d5a61ff835253a623ae9ee680
+  data.tar.gz: 577d01ac73bc69863cf41cf40051bf9d40d378be
 SHA512:
-  metadata.gz: 82c0a564358d3e814a966b5608d08e8b23541df4cc5015254517d43be96c96a0edbbc95a88393b0077649d0ba86f803b7acabab2ce64a332a41c9be398815ebd
-  data.tar.gz: 6e396e2a4884336894d9c894e78c0cf466e2023008a416b19ec86a8af70a9f5fa4b5eba0e10ec87c502c169457008765e129284db8c80c2408442547870205f3
+  metadata.gz: 80a7996018fb8202a8aa083e329482a202f6fb8e418db6ff7db957ee6bba9204cb7c79f5fb98ed2d4afcd3fa1c95743fc401648ff50702fc7c1657577ce01992
+  data.tar.gz: 7124cc5ade5ac68f92ee8e54bdc1e9005db4c0f81a9dd8087bdfedca36b77dceb9666db19f0694f2e7e98ec99996c3f8e23c53ddb13e1da41987d64b5e9bd4e3

data/README.markdown

@@ -1,15 +1,14 @@
 # YamlDb
 
-
 YamlDb is a database-independent format for dumping and restoring data.  It complements the the database-independent schema format found in db/schema.rb.  The data is saved into db/data.yml.
 
 This can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps.  Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.
 
 Any database that has an ActiveRecord adapter should work.  This gem is now Rails 3 only.  For Rails 2, clone and checkout the Rails2 branch.
 
-This is a fork of the ludicast version which makes sure schema tables are being dumped, too.
+_This is a fork of the ludicast version which makes sure schema tables are being dumped, too. Also, JSON is supported as an alternative serialization format._
 
-[![Build Status](https://secure.travis-ci.org/ludicast/yaml_db.png)](http://travis-ci.org/ludicast/yaml_db)
+[![Build Status](https://secure.travis-ci.org/zweitag/yaml_db.png)](http://travis-ci.org/zweitag/yaml_db)
 
 ## Installation
 
@@ -29,7 +28,7 @@ Further, there are tasks db:dump and db:load which do the entire database (the e
     rake db:data:dump_dir   ->   Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)
     rake db:data:load_dir   ->   Load contents of db/data_dir into database
 
-In addition, we have plugins whereby you can export your database to/from various formats.  We only deal with yaml and csv right now, but you can easily write tools for your own formats (such as Excel or XML).  To use another format, just load setting the "class"  parameter to the class you are using.  This defaults to "YamlDb::Helper" which is a refactoring of the old yaml_db code.  We'll shorten this to use class nicknames in a little bit.
+In addition, we have plugins whereby you can export your database to/from various formats.  We only deal with yaml and csv right now, but you can easily write tools for your own formats (such as Excel or XML).  To use another format, just load setting the "class"  parameter to the class you are using.  This defaults to "YamlDb::Helper" which is a refactoring of the old yaml_db code. Set it to "JsonDb::Helper" in order to use JSON serialization (much faster!) instead.
 
 ## Examples
 

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.3.2

data/lib/csv_db.rb

@@ -1,4 +1,5 @@
-#require 'FasterCSV'
+require 'csv'
+
 module CsvDb
   module Helper
     def self.loader
@@ -66,8 +67,8 @@ module CsvDb
 
       each_table_page(table) do |records|
         rows = SerializationHelper::Utils.unhash_records(records, column_names)
-        records.each do |record|
-          io.write(record.to_csv)
+        rows.each do |row|
+          io.write(row.to_csv)
         end
       end
     end
@@ -75,4 +76,4 @@ module CsvDb
   end
 
 
-end
+end

data/lib/json_db.rb

@@ -0,0 +1,74 @@
+require 'rubygems'
+require 'json'
+require 'json/add/core'
+require 'active_record'
+require 'serialization_helper'
+
+module JsonDb
+  module Helper
+    def self.loader
+      Load
+    end
+
+    def self.dumper
+      Dump
+    end
+
+    def self.extension
+      "json"
+    end
+  end
+
+  class Dump < SerializationHelper::Dump
+    def self.dump(io)
+      io.write '{'
+      super io
+      io.write '}'
+    end
+
+    def self.before_table(io, table)
+      io.write '"' + table + '": {'
+    end
+
+    def self.after_table(io, table)
+      if table == tables.last
+        io.write "}\n"
+      else
+        io.write "},\n"
+      end
+    end
+
+    def self.dump_table_columns(io, table)
+      io.write '"columns": ' + JSON.dump(table_column_names(table)) + ', '
+    end
+
+    def self.table_record_header(io)
+      io.write('"records": [ ')
+    end
+
+    def self.dump_table_records(io, table)
+      table_record_header(io)
+
+      column_names = table_column_names(table)
+
+      each_table_page(table) do |records, page_types|
+        rows = SerializationHelper::Utils.unhash_records(records, column_names)
+        io.write JSON.dump(rows)[1..-2]     # without opening and closing brackets
+        io.write ', ' unless :last.in? page_types
+      end
+
+      io.write ' ]'
+    end
+  end
+
+  class Load < SerializationHelper::Load
+    def self.load_documents(io, truncate = true)
+        JSON.load(io).tap do |json|
+          json.keys.each do |table_name|
+            next if json[table_name].nil?
+            load_table(table_name, json[table_name], truncate)
+          end
+        end
+    end
+  end
+end

data/lib/serialization_helper.rb

@@ -22,10 +22,16 @@ module SerializationHelper
         io = File.new "#{dirname}/#{table}.#{@extension}", "w"
         @dumper.before_table(io, table)
         @dumper.dump_table io, table
-        @dumper.after_table(io, table)         
+        @dumper.after_table(io, table)
       end
     end
 
+    def dump_to_io(io)
+      disable_logger
+      @dumper.dump(io)
+      reenable_logger
+    end
+
     def load(filename, truncate = true)
       disable_logger
       @loader.load(File.new(filename, "r"), truncate)
@@ -38,7 +44,13 @@ module SerializationHelper
           next
         end
         @loader.load(File.new("#{dirname}/#{filename}", "r"), truncate)
-      end   
+      end
+    end
+
+    def load_from_io(io, truncate = true)
+      disable_logger
+      @loader.load(io, truncate)
+      reenable_logger
     end
 
     def disable_logger
@@ -67,6 +79,7 @@ module SerializationHelper
     end
 
     def self.load_table(table, data, truncate = true)
+      return if table == 'ar_internal_metadata'
       column_names = data['columns']
       if truncate
         truncate_table(table)
@@ -75,16 +88,19 @@ module SerializationHelper
       reset_pk_sequence!(table)
     end
 
-    def self.load_records(table, column_names, records)
+    def self.load_records(table, column_names, records, records_per_page=1000)
       if column_names.nil?
         return
       end
       columns = column_names.map{|cn| ActiveRecord::Base.connection.columns(table).detect{|c| c.name == cn}}
       quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',')
       quoted_table_name = SerializationHelper::Utils.quote_table(table)
-      records.each do |record|
-        quoted_values = record.zip(columns).map{|c| ActiveRecord::Base.connection.quote(c.first, c.last)}.join(',')
-        ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})")
+      
+      0.step(records.count-1, records_per_page) do |offset|
+        all_quoted_values = records[offset, records_per_page].map do |record|
+          '(' + record.zip(columns).map{|c| ActiveRecord::Base.connection.quote(c.first, c.last)}.join(',') + ')'
+        end.join(', ')
+        ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES #{all_quoted_values}")
       end
     end
 
@@ -99,41 +115,59 @@ module SerializationHelper
 
   module Utils
 
-    def self.unhash(hash, keys)
-      keys.map { |key| hash[key] }
-    end
-
     def self.unhash_records(records, keys)
-      records.each_with_index do |record, index|
-        records[index] = unhash(record, keys)
+      records.map do |record|
+        keys.map { |key| record[key] }
       end
-
-      records
     end
 
     def self.convert_booleans(records, columns)
-      records.each do |record|
+      records.map do |record|
         columns.each do |column|
           next if is_boolean(record[column])
           record[column] = convert_boolean(record[column])
         end
+        record
+      end
+    end
+
+    def self.convert_jsons(records, columns)
+      records.map do |record|
+        columns.each do |column|
+          next if is_json(record[column])
+          record[column] = convert_json(record[column])
+        end
+        record
       end
-      records
     end
 
     def self.convert_boolean(value)
       ['t', '1', true, 1].include?(value)
     end
 
+    def self.convert_json(value)
+      return nil if value.nil?
+      JSON.parse(value)
+    end
+
     def self.boolean_columns(table)
       columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
       columns.map { |c| c.name }
     end
 
+    def self.json_columns(table)
+      columns = ActiveRecord::Base.connection.columns(table).select { |c| c.sql_type == 'json' }
+      columns.map { |c| c.name }
+    end
+
     def self.is_boolean(value)
       value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
     end
 
+    def self.is_json(value)
+      value.kind_of?(Hash) or value.kind_of?(Array)
+    end
+
     def self.quote_table(table)
       ActiveRecord::Base.connection.quote_table_name(table)
     end
@@ -178,13 +212,20 @@ module SerializationHelper
       pages = (total_count.to_f / records_per_page).ceil - 1
       id = table_column_names(table).first
       boolean_columns = SerializationHelper::Utils.boolean_columns(table)
+      json_columns = SerializationHelper::Utils.json_columns(table)
       quoted_table_name = SerializationHelper::Utils.quote_table(table)
 
-      (0..pages).to_a.each do |page|
+      (0..pages).to_a.each_with_index do |page, index|
         query = Arel::Table.new(table).order(id).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*'))
         records = ActiveRecord::Base.connection.select_all(query)
         records = SerializationHelper::Utils.convert_booleans(records, boolean_columns)
-        yield records
+        records = SerializationHelper::Utils.convert_jsons(records, json_columns)
+
+        page_types = []
+        page_types << :first if index == 0
+        page_types << :last if index == pages
+
+        yield records, page_types
       end
     end
 

data/lib/tasks/yaml_db_tasks.rake

@@ -1,3 +1,5 @@
+require 'zlib'
+
 namespace :db do
   desc "Dump schema and data to db/schema.rb and db/data.yml"
   task(:dump => [ "db:schema:dump", "db:data:dump" ])
@@ -16,30 +18,45 @@ namespace :db do
 
     desc "Dump contents of database to db/data.extension (defaults to yaml)"
     task :dump => :environment do
-      format_class = ENV['class'] || "YamlDb::Helper"
-      helper = format_class.constantize
       SerializationHelper::Base.new(helper).dump db_dump_data_file helper.extension
     end
 
     desc "Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)"
     task :dump_dir => :environment do
-      format_class = ENV['class'] || "YamlDb::Helper"
       dir = ENV['dir'] || "#{Time.now.to_s.gsub(/ /, '_')}"
-      SerializationHelper::Base.new(format_class.constantize).dump_to_dir dump_dir("/#{dir}")
+    end
+
+    desc "Dump gzipped contents of database to stdout"
+    task :dump_gzipped_stdout => :environment do
+      gz = Zlib::GzipWriter.new($stdout.dup)
+      SerializationHelper::Base.new(helper).dump_to_io gz
+      gz.close
     end
 
     desc "Load contents of db/data.extension (defaults to yaml) into database"
     task :load => :environment do
-      format_class = ENV['class'] || "YamlDb::Helper"
-      helper = format_class.constantize
       SerializationHelper::Base.new(helper).load(db_dump_data_file helper.extension)
     end
 
     desc "Load contents of db/data_dir into database"
     task :load_dir  => :environment do
       dir = ENV['dir'] || "base"
-      format_class = ENV['class'] || "YamlDb::Helper"
-      SerializationHelper::Base.new(format_class.constantize).load_from_dir dump_dir("/#{dir}")
+      SerializationHelper::Base.new(helper).load_from_dir dump_dir("/#{dir}")
+    end
+
+    desc "Load gzipped contents from stdin"
+    task :load_gzipped_stdin => :environment do
+      gz = Zlib::GzipReader.new($stdin.dup)
+      SerializationHelper::Base.new(helper).load_from_io gz
+      gz.close
+    end
+
+    def helper
+      @helper ||= begin
+        format_class = ENV['class'] || "YamlDb::Helper"
+        require format_class.split('::').first.underscore
+        format_class.constantize
+      end
     end
   end
 end

data/lib/yaml_db.rb

@@ -45,7 +45,7 @@ module YamlDb
 
       each_table_page(table) do |records|
         rows = SerializationHelper::Utils.unhash_records(records, column_names)
-        io.write(YamlDb::Utils.chunk_records(records))
+        io.write(YamlDb::Utils.chunk_records(rows))
       end
     end
 

data/spec/base.rb

@@ -2,3 +2,5 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'rspec'
 require 'yaml_db'
+require 'json_db'
+require 'csv_db'

data/spec/json_dump_spec.rb

@@ -0,0 +1,81 @@
+require File.dirname(__FILE__) + '/base'
+require 'date'
+
+describe JsonDb::Dump do
+
+	before do
+		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
+		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
+		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
+		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
+		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
+	end
+
+	before(:each) do   
+	  File.stub!(:new).with('dump.json', 'w').and_return(StringIO.new)
+	  @io = StringIO.new
+	end
+
+  it "should dump a valid json document with correct data" do
+		ActiveRecord::Base.connection.stub!(:columns).and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
+
+    JsonDb::Dump.dump(@io)
+    @io.rewind
+    expect { @json = JSON.load @io }.not_to raise_error
+
+    @json['mytable']['columns'].count.should == 2
+    @json['mytable']['records'].should match_array [[1, 2], [3, 4]]
+  end
+
+	it "should return a formatted string" do
+		JsonDb::Dump.table_record_header(@io)
+		@io.rewind
+		@io.read.should == '"records": [ '
+	end
+
+	it "should return a json string that contains column names" do
+		JsonDb::Dump.stub!(:table_column_names).with('mytable').and_return([ 'a', 'b' ])
+		JsonDb::Dump.dump_table_columns(@io, 'mytable')
+		@io.rewind
+    @io.read.should == '"columns": ["a","b"], '
+  end
+
+	it "should return dump the records for a table in json to a given io stream" do
+		JsonDb::Dump.dump_table_records(@io, 'mytable')
+		@io.rewind
+		@io.read.should == '"records": [ [1,2],[3,4] ]'
+  end
+
+  it 'should dump a valid json document for more than 1000 records' do
+		ActiveRecord::Base.connection.stub!(:tables).and_return(['mytable'])
+		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"1001"})
+		ActiveRecord::Base.connection.stub!(:select_all).and_return([{'a'=>1, 'b'=>2}] * 1000, [{'a'=>1, 'b'=>2}])
+
+    JsonDb::Dump.dump(@io)
+    @io.rewind
+    expect { @json = JSON.load @io }.not_to raise_error
+
+    @json['mytable']['columns'].count.should == 2
+    @json['mytable']['records'].count.should == 1001
+    @json['mytable']['records'].should match_array([[1, 2]] * 1001)
+  end
+
+  it 'should dump datetime objects using custom Ruby serialization' do
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('datetime', name: 'datetime', type: :datetime, sql_type: 'datetime')])
+		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"1"})
+		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'datetime' => DateTime.new(2014, 1, 1, 12, 20, 00) } ])
+		JsonDb::Dump.dump_table_records(@io, 'mytable')
+		@io.rewind
+		@io.read.should == '"records": [ [{"json_class":"DateTime","y":2014,"m":1,"d":1,"H":12,"M":20,"S":0,"of":"0/1","sg":2299161.0}] ]'
+  end
+
+  it 'should correctly serialize json columns' do
+    ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('json', name: 'json', type: :json, sql_type: 'json')])
+    ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"1"})
+    ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'json' => '[{"a":1},{"b":2}]' } ])
+    JsonDb::Dump.dump_table_records(@io, 'mytable')
+    @io.rewind
+    @io.read.should == '"records": [ [[{"a":1},{"b":2}]] ]'
+  end
+end

data/spec/json_load_spec.rb

@@ -0,0 +1,46 @@
+require File.dirname(__FILE__) + '/base'
+require 'active_support/core_ext/kernel/debugger'
+
+describe JsonDb::Load do
+	before do
+		SerializationHelper::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
+
+		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
+		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
+		ActiveRecord::Base.connection.stub!(:transaction).and_yield
+	end
+
+	before(:each) do
+		@io = StringIO.new
+    end
+    
+
+	it "should call load_table for each table in the file" do
+		JSON.should_receive(:load).with(@io).and_return({ 'mytable' => { 
+					'columns' => [ 'a', 'b' ], 
+					'records' => [[1, 2], [3, 4]] 
+				} } )
+		JsonDb::Load.should_receive(:load_table).with('mytable', { 'columns' => [ 'a', 'b' ], 'records' => [[1, 2], [3, 4]] },true)
+		JsonDb::Load.load(@io)
+	end
+
+	it "should not call load_table when the table in the file contains no records" do
+		JSON.should_receive(:load).with(@io).and_return({ 'mytable' => nil })
+		JsonDb::Load.should_not_receive(:load_table)
+		JsonDb::Load.load(@io)
+	end
+
+  it "should load custom Ruby serialized datetime objects" do
+    @io = StringIO.new <<-EOYAML
+    {
+      "mytable": { 
+        "columns": [ "datetime" ], 
+        "records": [ [{"json_class":"DateTime","y":2014,"m":1,"d":1,"H":12,"M":20,"S":0,"of":"0/1","sg":2299161.0}] ]
+      }
+    }
+    EOYAML
+
+		JsonDb::Load.should_receive(:load_table).with('mytable', { 'columns' => [ 'datetime' ], 'records' => [ [DateTime.new(2014, 1, 1, 12, 20)] ] },true)
+		JsonDb::Load.load(@io)
+  end
+end

data/spec/serialization_helper_dump_spec.rb

@@ -2,64 +2,71 @@ require File.dirname(__FILE__) + '/base'
 
 describe SerializationHelper::Dump do
 
-	before do
-		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
-		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
-		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', :name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
-		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
-		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
-		SerializationHelper::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
-	end
+  before do
+    silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
+    ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
+    ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
+    ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([
+      mock('a', name: 'a', type: :string, sql_type: 'text'),
+      mock('b', name: 'b', type: :string, sql_type: 'text'),
+      mock('json', name: 'json', type: :json, sql_type: 'json')
+    ])
+    ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
+    ActiveRecord::Base.connection.stub!(:select_all).and_return([
+      { 'a' => 1, 'b' => 2, 'json' => '[{"c": 3}]' },
+      { 'a' => 3, 'b' => 4, 'json' => nil }
+    ])
+    SerializationHelper::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
+  end
 
-	before(:each) do   
-	  File.stub!(:new).with('dump.yml', 'w').and_return(StringIO.new)
-	  @io = StringIO.new
-	end
+  before(:each) do
+    File.stub!(:new).with('dump.yml', 'w').and_return(StringIO.new)
+    @io = StringIO.new
+  end
 
-	it "should return a list of column names" do
-		SerializationHelper::Dump.table_column_names('mytable').should == [ 'a', 'b' ]
-	end
+  it "should return a list of column names" do
+    SerializationHelper::Dump.table_column_names('mytable').should == [ 'a', 'b', 'json' ]
+  end
 
-	it "should return a list of tables including the rails schema table" do
-		SerializationHelper::Dump.tables.should == ['mytable', 'schema_info', 'schema_migrations']
-	end
+  it "should return a list of tables including the rails schema table" do
+    SerializationHelper::Dump.tables.should == ['mytable', 'schema_info', 'schema_migrations']
+  end
 
-	it "should return the total number of records in a table" do
-		SerializationHelper::Dump.table_record_count('mytable').should == 2
-	end
+  it "should return the total number of records in a table" do
+    SerializationHelper::Dump.table_record_count('mytable').should == 2
+  end
 
-	it "should return all records from the database and return them when there is only 1 page" do
-		SerializationHelper::Dump.each_table_page('mytable') do |records|
-			records.should == [ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]
-		end
-	end
-
-	it "should paginate records from the database and return them" do
-		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 } ], [ { 'a' => 3, 'b' => 4 } ])
-
-		records = [ ]
-		SerializationHelper::Dump.each_table_page('mytable', 1) do |page|
-			page.size.should == 1
-			records.concat(page)
-		end
-
-		records.should == [ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]
-	end
+  it "should return all records from the database and return them when there is only 1 page" do
+    SerializationHelper::Dump.each_table_page('mytable') do |records|
+      records.should == [ { 'a' => 1, 'b' => 2, 'json' => [{'c' => 3}] }, { 'a' => 3, 'b' => 4, 'json' => nil } ]
+    end
+  end
 
-	it "should dump a table's contents to yaml" do
-		SerializationHelper::Dump.should_receive(:dump_table_columns)
-		SerializationHelper::Dump.should_receive(:dump_table_records)
-		SerializationHelper::Dump.dump_table(@io, 'mytable')
-	end
+  it "should paginate records from the database and return them" do
+    ActiveRecord::Base.connection.stub!(:select_all).and_return(
+      [ { 'a' => 1, 'b' => 2, 'json' => '[{"c": 3}]' } ],
+      [ { 'a' => 3, 'b' => 4, 'json' => nil } ]
+    )
 
-	it "should not dump a table's contents when the record count is zero" do
-		SerializationHelper::Dump.stub!(:table_record_count).with('mytable').and_return(0)
-		SerializationHelper::Dump.should_not_receive(:dump_table_columns)
-		SerializationHelper::Dump.should_not_receive(:dump_table_records)
-		SerializationHelper::Dump.dump_table(@io, 'mytable')
+    records = [ ]
+    SerializationHelper::Dump.each_table_page('mytable', 1) do |page|
+      page.size.should == 1
+      records.concat(page)
     end
 
+    records.should == [ { 'a' => 1, 'b' => 2, 'json' => [{'c' => 3}] }, { 'a' => 3, 'b' => 4, 'json' => nil } ]
+  end
 
+  it "should dump a table's contents to yaml" do
+    SerializationHelper::Dump.should_receive(:dump_table_columns)
+    SerializationHelper::Dump.should_receive(:dump_table_records)
+    SerializationHelper::Dump.dump_table(@io, 'mytable')
+  end
 
+  it "should not dump a table's contents when the record count is zero" do
+    SerializationHelper::Dump.stub!(:table_record_count).with('mytable').and_return(0)
+    SerializationHelper::Dump.should_not_receive(:dump_table_columns)
+    SerializationHelper::Dump.should_not_receive(:dump_table_records)
+    SerializationHelper::Dump.dump_table(@io, 'mytable')
+  end
 end

data/spec/serialization_helper_load_spec.rb

@@ -45,8 +45,7 @@ describe SerializationHelper::Load do
         ActiveRecord::Base.connection.stub!(:quote).with(2, mcb).and_return("'2'")
         ActiveRecord::Base.connection.stub!(:quote).with(3, mca).and_return("'3'")
         ActiveRecord::Base.connection.stub!(:quote).with(4, mcb).and_return("'4'")
-        ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('1','2')")
-        ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('3','4')")
+        ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('1','2'), ('3','4')")
 
         SerializationHelper::Load.load_records('mytable', ['a', 'b'], [[1, 2], [3, 4]])
     end
@@ -61,8 +60,7 @@ describe SerializationHelper::Load do
         ActiveRecord::Base.connection.stub!(:quote).with(2, mccount).and_return("'2'")
         ActiveRecord::Base.connection.stub!(:quote).with(3, mca).and_return("'3'")
         ActiveRecord::Base.connection.stub!(:quote).with(4, mccount).and_return("'4'")
-        ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('1','2')")
-        ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('3','4')")
+        ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('1','2'), ('3','4')")
 
         SerializationHelper::Load.load_records('mytable', ['a', 'count'], [[1, 2], [3, 4]])
     end

data/spec/serialization_utils_spec.rb

@@ -6,10 +6,6 @@ describe SerializationHelper::Utils, " convert records utility method" do
 		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
 	end
 
-	it "returns an array of hash values using an array of ordered keys" do
-		SerializationHelper::Utils.unhash({ 'a' => 1, 'b' => 2 }, [ 'b', 'a' ]).should == [ 2, 1 ]
-    end
-
 	it "should unhash each hash an array using an array of ordered keys" do
 		SerializationHelper::Utils.unhash_records([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ], [ 'b', 'a' ]).should == [ [ 2, 1 ], [ 4, 3 ] ]
 	end

data/spec/yaml_dump_spec.rb

@@ -6,7 +6,7 @@ describe YamlDb::Dump do
 		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
 		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
 		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a',:name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
 		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
 		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
 		YamlDb::Utils.stub!(:quote_table).with('mytable').and_return('mytable')

data/yaml_db.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "yaml_db_with_schema_tables"
-  s.version = "0.2.4"
+  s.version = "0.3.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adam Wiggins", "Orion Henry", "Martin Honermeyer"]
-  s.date = "2014-01-20"
+  s.date = "2017-12-04"
   s.description = "\nYamlDb is a database-independent format for dumping and restoring data.  It complements the the database-independent schema format found in db/schema.rb.  The data is saved into db/data.yml.\nThis can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps.  Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.\nAny database that has an ActiveRecord adapter should work\n"
   s.email = "martin.honermeyer@zweitag.de"
   s.extra_rdoc_files = [
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
     "about.yml",
     "init.rb",
     "lib/csv_db.rb",
+    "lib/json_db.rb",
     "lib/serialization_helper.rb",
     "lib/tasks/yaml_db_tasks.rake",
     "lib/yaml_db.rb",
@@ -32,6 +33,8 @@ Gem::Specification.new do |s|
     "spec/serialization_helper_dump_spec.rb",
     "spec/serialization_helper_load_spec.rb",
     "spec/serialization_utils_spec.rb",
+    "spec/json_dump_spec.rb",
+    "spec/json_load_spec.rb",
     "spec/yaml_dump_spec.rb",
     "spec/yaml_load_spec.rb",
     "spec/yaml_utils_spec.rb",
@@ -40,7 +43,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/zweitag/yaml_db"
   s.require_paths = ["lib"]
   s.rubygems_version = "1.8.17"
-  s.summary = "yaml_db allows export/import of database into/from yaml files"
+  s.summary = "yaml_db allows export/import of database into/from yaml/json files"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yaml_db_with_schema_tables
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.2
 platform: ruby
 authors:
 - Adam Wiggins
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-20 00:00:00.000000000 Z
+date: 2017-12-04 00:00:00.000000000 Z
 dependencies: []
 description: |2
 
@@ -23,18 +23,21 @@ extensions: []
 extra_rdoc_files:
 - README.markdown
 files:
-- .travis.yml
+- ".travis.yml"
 - README.markdown
 - Rakefile
 - VERSION
 - about.yml
 - init.rb
 - lib/csv_db.rb
+- lib/json_db.rb
 - lib/serialization_helper.rb
 - lib/tasks/yaml_db_tasks.rake
 - lib/yaml_db.rb
 - lib/yaml_db_with_schema_tables.rb
 - spec/base.rb
+- spec/json_dump_spec.rb
+- spec/json_load_spec.rb
 - spec/serialization_helper_base_spec.rb
 - spec/serialization_helper_dump_spec.rb
 - spec/serialization_helper_load_spec.rb
@@ -52,19 +55,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.6.7
 signing_key: 
 specification_version: 3
-summary: yaml_db allows export/import of database into/from yaml files
+summary: yaml_db allows export/import of database into/from yaml/json files
 test_files: []
-has_rdoc: