database_sanitizer 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ae899e9c69e083b9ad8adb9f427a384c898621d
-  data.tar.gz: 704c9bb2b7ee445b459078edf86fd6198023cd28
+  metadata.gz: 1e3c8cc11038ed94c46e29f7b66b1289372bed89
+  data.tar.gz: b9c1d829f7635a7ea1a53db9cce6fb04adb0e4c9
 SHA512:
-  metadata.gz: 0045968c62ea497b1ba352b60c5eb58c629ce2e1fcaf5480f73afabd68c8a8ba234b7a1e62a16e3563533a9e4d4e8f0fc54446af7efb09c6f0a57b6a62306d09
-  data.tar.gz: c1ebeac6a04345da7edbef0cffb4beee17276e47d6bdd8add408f1a6ec9a7e4c31beac21a197667c369eac31773857996758e3899a9a87f9891da0260c4f88be
+  metadata.gz: 96c5f9a2548522100570d8125520a8997bf6dc7c7b7e36be4c22616a3ac41d8e86e10cf2e365cd18b2f2da5d2f7e1b7884a35972b8764c6f868fab4ebc60da88
+  data.tar.gz: 52f4150d8cd04a646d18f5a0fb5aba5df4163769377c0b789fa0e3d2831ad2cae1d1fc1f128744f2b7238708082f8831dd678e603427bda4704421c695da1176

data/lib/database_sanitizer.rb

@@ -3,6 +3,7 @@ require 'active_record/comments'
 require 'progress'
 
 module DatabaseSanitizer
+  CHUNK_SIZE = 500
   class Source < ActiveRecord::Base
   end
 end
@@ -42,22 +43,35 @@ module DatabaseSanitizer
       end
     end
 
+    def get_chunks table
+      conn = Source.connection
+      query = "SELECT count(*) FROM #{conn.quote_table_name table}"
+      pg_query = "SELECT reltuples FROM pg_class WHERE relname=#{conn.quote table}"
+      res = conn.adapter_name == 'PostgreSQL' ? (conn.exec_query(pg_query) rescue false) : false
+      res ||= conn.exec_query(query)
+      res.rows[0][0].to_i % CHUNK_SIZE + 1
+    end
+
     def export src, dest, opts={}
       duplicate_schema opts[:schema]
       tables = (opts[:tables] || src.tables.collect(&:to_s)) - (opts[:exclude] || [])
       transformers = read_comments dest, tables
-
       max_tbl_name_len = transformers.keys.map(&:length).sort.last || 0
+
       tables.with_progress('Exporting').each do |table|
-        result = src.exec_query "SELECT * FROM #{table}"
-        cols = result.columns.join ','
-        dest.transaction do
-          result.rows.with_progress(table.rjust max_tbl_name_len).each_with_index do |src_row, row_i|
-            values = result.columns.each_with_index.map do |col, col_i|
-              transformer = transformers[table.to_sym][col.to_sym]
-              dest.quote transformer ? transformer.(row_i, src_row[col_i]) : src_row[col_i]
+        q_table = dest.quote_table_name table
+        query = "SELECT * FROM #{q_table} LIMIT #{CHUNK_SIZE} OFFSET "
+        get_chunks(table).times_with_progress(table.rjust max_tbl_name_len) do |chunk_i|
+          result = src.exec_query query + (chunk_i*CHUNK_SIZE).to_s
+          cols = result.columns.map { |col| dest.quote_column_name col }.join ','
+          dest.transaction do
+            result.rows.with_progress.each_with_index do |src_row, row_i|
+              values = result.columns.each_with_index.map do |col, col_i|
+                transformer = transformers[table.to_sym][col.to_sym]
+                dest.quote transformer ? transformer.(row_i, src_row[col_i]) : src_row[col_i]
+              end
+              dest.insert_sql "INSERT INTO #{q_table} (#{cols}) VALUES (#{values.join ','})"
             end
-            dest.insert_sql "INSERT INTO #{dest.quote_table_name table} (#{cols}) VALUES (#{values.join ','})"
           end
         end
       end

data/lib/database_sanitizer/version.rb

@@ -1,3 +1,3 @@
 module DatabaseSanitizer
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
 end

data/spec/database_exporter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe DatabaseExporter do
+describe DatabaseSanitizer do
   describe '#extract_transformer', nodb: true do
     context 'should return nil for no transformer' do
       ['no tag comment', nil, '', 'sanitize no tag'].each do |comment|

data/spec/spec_helper.rb

@@ -3,7 +3,7 @@ $: << File.expand_path('../lib', __FILE__)
 require 'yaml'
 require 'pry'
 
-require 'database_exporter'
+require 'database_sanitizer'
 
 DBCONF = YAML::load(IO.read(File.expand_path('../config/database.yml', __FILE__)))
 ENV['DB'] ||= 'postgres'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: database_sanitizer
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Marton Somogyi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-26 00:00:00.000000000 Z
+date: 2014-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler