sp-squealer 1.0

data/lib/squealer/hash.rb

@@ -0,0 +1,7 @@
+class Hash
+  def method_missing(name, *args, &block)
+    super if args.size > 0 || block_given?
+    #TODO: Warn if key doesn't exist - it's probably a typo in their squealer script
+    self[name.to_s]
+  end
+end

data/lib/squealer/object.rb

@@ -0,0 +1,34 @@
+class Object
+
+  def target(table_name, &block)
+    Squealer::Target.new(Squealer::Database.instance.export, table_name, &block)
+  end
+
+  def assign(column_name, &block)
+    Squealer::Target.current.assign(column_name, &block)
+  end
+
+  def import(*args)
+    if args.length > 0
+      Squealer::Database.instance.import_from(*args)
+    else
+      Squealer::Database.instance.import
+    end
+  end
+
+  def export(*args)
+    if args.length > 0
+      Squealer::Database.instance.export_to(*args)
+    else
+      Squealer::Database.instance.export
+    end
+  end
+
+end
+
+class NilClass
+  include Enumerable
+  def each
+    []
+  end
+end

data/lib/squealer/progress_bar.rb

@@ -0,0 +1,122 @@
+module Squealer
+  class ProgressBar
+
+    @@progress_bar = nil
+
+    def self.new(*args)
+      if @@progress_bar
+        nil
+      else
+        @@progress_bar = super
+      end
+    end
+
+    def initialize(total)
+      @total = total
+      @ticks = 0
+
+      @progress_bar_width = 50
+      @count_width = total.to_s.size
+    end
+
+    def start
+      @start_time = clock
+      @emitter = start_emitter if total > 0
+      self
+    end
+
+    def finish
+      @end_time = clock
+      @emitter.wakeup.join if @emitter
+      @@progress_bar = nil
+    end
+
+    def tick
+      @ticks += 1
+    end
+
+    private
+
+    def start_emitter
+      Thread.new do
+        emit
+        sleep(1) and emit until done?
+      end
+    end
+
+    def emit
+      format = "\r[%-#{progress_bar_width}s] %#{count_width}i/%i (%i%%) [%i/s]"
+      console.print format % [progress_markers, ticks, total, percentage, tps]
+      emit_final if done?
+    end
+
+    def emit_final
+      console.puts
+
+      console.puts "Start: #{start_time}"
+      console.puts "End: #{end_time}"
+      console.puts "Duration: #{duration}"
+    end
+
+    def clock
+      Time.now
+    end
+
+    def done?
+      ticks >= total || end_time
+    end
+
+    def start_time
+      @start_time
+    end
+
+    def end_time
+      @end_time
+    end
+
+    def ticks
+      @ticks
+    end
+
+    def tps
+      elapsed_secs = (clock - start_time) / 60
+      (ticks / elapsed_secs).ceil
+    rescue
+      0
+    end
+
+    def total
+      @total
+    end
+
+    def percentage
+      ((ticks.to_f / total) * 100).floor
+    end
+
+    def progress_markers
+      "=" * ((ticks.to_f / total) * progress_bar_width).floor
+    end
+
+    def console
+      $stderr
+    end
+
+    def progress_bar_width
+      @progress_bar_width
+    end
+
+    def count_width
+      @count_width
+    end
+
+    def total_time
+      @end_time - @start_time
+    end
+
+    def duration
+      duration = Time.at(total_time).utc
+      duration.strftime("%H:%M:%S.#{duration.usec}")
+    end
+
+  end
+end

data/lib/squealer/target.rb

@@ -0,0 +1,177 @@
+require 'delegate'
+require 'singleton'
+
+#TODO: Use logger and log throughout
+
+module Squealer
+  class Target
+
+    def self.current
+      Queue.instance.current
+    end
+
+    def initialize(database_connection, table_name, &block)
+      raise BlockRequired, "Block must be given to target (otherwise, there's no work to do)" unless block_given?
+      raise ArgumentError, "Table name must be supplied" if table_name.to_s.strip.empty?
+
+      @dbc = database_connection
+      @table_name = table_name.to_s
+      @binding = block.binding
+
+      verify_table_name_in_scope
+      @row_id = infer_row_id
+      @column_names = []
+      @column_values = []
+      @sql = ''
+
+      target(&block)
+    end
+
+    def sql
+      @sql
+    end
+
+    def assign(column_name, &block)
+      @column_names << column_name
+      if block_given?
+        @column_values << yield
+      else
+        @column_values << infer_value(column_name, @binding)
+      end
+    end
+
+
+    private
+
+    def infer_row_id
+      (
+        (eval "#{@table_name}[:_id]", @binding, __FILE__, __LINE__) ||
+        (eval "#{@table_name}['_id']", @binding, __FILE__, __LINE__)
+      ).to_s
+    end
+
+    def verify_table_name_in_scope
+      table = eval "#{@table_name}", @binding, __FILE__, __LINE__
+      raise ArgumentError, "The variable '#{@table_name}' is not a hashmap" unless table.is_a? Hash
+      raise ArgumentError, "The hashmap '#{@table_name}' must have an '_id' key" unless table.has_key?('_id') || table.has_key?(:_id)
+    rescue NameError
+      raise NameError, "A variable named '#{@table_name}' must be in scope, and reference a hashmap with at least an '_id' key."
+    end
+
+
+    def infer_value(column_name, binding)
+      value = eval "#{@table_name}.#{column_name}", binding, __FILE__, __LINE__
+      unless value
+        name = column_name.to_s
+        if name =~ /_id$/
+          related = name[0..-4]  #strip "_id"
+          value = eval "#{related}._id", binding, __FILE__, __LINE__
+        end
+      end
+      value
+    end
+
+    def target
+      Queue.instance.push(self)
+
+      yield self
+
+      insert_statement = %{INSERT INTO "#{@table_name}"}
+      insert_statement << %{ (#{pk_name}#{column_names}) VALUES ('#{@row_id}'#{column_value_markers})}
+      if Database.instance.upsertable?
+        insert_statement << %{ ON DUPLICATE KEY UPDATE #{column_markers}}
+        @sql = insert_statement
+      else
+        update_statement = %{UPDATE "#{@table_name}" SET #{column_markers} WHERE #{pk_name}='#{@row_id}'}
+        process_sql(update_statement)
+        @sql = update_statement + "; " + insert_statement
+      end
+
+      process_sql(insert_statement)
+
+      Queue.instance.pop
+    end
+
+    def self.targets
+      @@targets
+    end
+
+    def targets
+      @@targets
+    end
+
+    def process_sql(sql)
+      values = Database.instance.upsertable? ? typecast_values * 2 : typecast_values
+      execute_sql(sql, values)
+    end
+
+    def execute_sql(sql, values)
+      @dbc.create_command(sql).execute_non_query(*values)
+    rescue DataObjects::IntegrityError
+      raise "Failed to execute statement: #{sql} with #{values.inspect}.\nOriginal Exception was: #{$!.to_s}" if Database.instance.upsertable?
+    rescue
+      raise "Failed to execute statement: #{sql} with #{values.inspect}.\nOriginal Exception was: #{$!.to_s}"
+    end
+
+    def pk_name
+      'id'
+    end
+
+    def column_names
+      return if @column_names.size == 0
+      ",#{@column_names.map { |name| quote_identifier(name) }.join(',')}"
+    end
+
+    def column_values
+      @column_values
+    end
+
+    def column_value_markers
+      return if @column_names.size == 0
+      result = ""
+      @column_names.size.times { result << ',?'}
+      result
+    end
+
+    def column_markers
+      return if @column_names.size == 0
+      result = ""
+      @column_names.each {|k| result << "#{quote_identifier(k)}=?," }
+      result.chop
+    end
+
+    def typecast_values
+      column_values.map do |value|
+        case value
+        when Array
+          value.join(",")
+        when BSON::ObjectId
+          value.to_s
+        else
+          value
+        end
+      end
+    end
+
+    def quote_identifier(name)
+      %{"#{name}"}
+    end
+
+    class Queue < DelegateClass(Array)
+      include Singleton
+
+      def current
+        last
+      end
+
+      protected
+
+      def initialize
+        super([])
+      end
+    end
+
+    class BlockRequired < ArgumentError; end
+
+  end
+end

data/lib/tasks/jeweler.rake

@@ -0,0 +1,15 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "squealer"
+    gemspec.summary = "Document-oriented to Relational database exporter"
+    gemspec.description = "Exports mongodb to mysql. More later."
+    gemspec.email = "joshua.graham@grahamis.com"
+    gemspec.homepage = "http://github.com/deltiscere/squealer/"
+    gemspec.authors = ["Josh Graham", "Durran Jordan"]
+    gem.add_dependency('mysql', '>= 2.8.1')
+    gem.add_dependency('mongo', '>= 0.18.3')
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/spec/integration/export_a_record_spec.rb

@@ -0,0 +1,136 @@
+require File.expand_path(File.dirname(__FILE__) + '/./spec_helper_dbms')
+
+describe "Exporting" do
+  let(:databases) { Squealer::Database.instance }
+  let(:today) { Date.today }
+
+  def squeal_basic_users_document(user=users_document)
+    target(:user) do
+      assign(:name)
+      assign(:organization_id)
+      assign(:dob)
+      assign(:gender)
+      assign(:foreign)
+      assign(:dull)
+      assign(:symbolic)
+      assign(:interests)
+    end
+  end
+
+  def squeal_users_document_with_activities(user=users_document)
+    target(:user) do
+      assign(:name)
+      assign(:organization_id)
+      assign(:dob)
+      assign(:gender)
+      assign(:foreign)
+      assign(:dull)
+      assign(:symbolic)
+      assign(:interests)
+
+      user.activities.each do |activity|
+        target(:activity) do
+          assign(:name)
+          assign(:due_date)
+        end
+      end
+    end
+  end
+
+  let :users_document do
+    { :_id => 'ABCDEFGHIJKLMNOPQRSTUVWX',
+      'name' => 'Test User', 'dob' => as_time(Date.parse('04-Jul-1776')), 'gender' => 'M',
+      'foreign' => true,
+      'dull' => false,
+      'symbolic' => :of_course,
+      'interests' => ['health', 'education'],
+      'organization_id' => '123456789012345678901234',
+      'activities' => [
+        { :_id => 'a1', 'name' => 'Be independent', 'due_date' => as_time(today + 1) },
+        { :_id => 'a2', 'name' => 'Fight each other', 'due_date' => as_time(today + 7) }
+      ]
+    }
+  end
+
+  let :first_users_record do
+    dbc = databases.export
+    reader = dbc.create_command(%{SELECT * FROM "user"}).execute_reader
+    result = reader.each { |x| break x }
+    reader.close
+    result
+  end
+
+  let :first_activity_record do
+    dbc = databases.export
+    reader = dbc.create_command(%{SELECT * FROM "activity"}).execute_reader
+    result = reader.each { |x| break x }
+    reader.close
+    result
+  end
+
+  context "a new record" do
+    it "saves the data correctly" do
+      squeal_basic_users_document
+      result = first_users_record
+
+      result['name'].should == 'Test User'
+
+      result['dob'].mday.should == 4
+      result['dob'].mon.should == 7
+      result['dob'].year.should == 1776
+
+      result['gender'].should == 'M'
+
+      result['foreign'].should be_true
+      result['dull'].should be_false
+
+      result['symbolic'].should == :of_course.to_s
+
+      result['interests'].should == 'health,education'
+    end
+
+    it "saves embedded documents correctly" do
+      squeal_users_document_with_activities
+      result = first_activity_record
+
+      result['name'].should == 'Be independent'
+      result['due_date'].mday.should == (today + 1).mday
+      result['due_date'].mon.should == (today + 1).mon
+      result['due_date'].year.should == (today + 1).year
+    end
+  end
+
+  context "an existing record" do
+    it "updates the data correctly" do
+      squeal_basic_users_document
+      squeal_basic_users_document(users_document.merge('foreign' => false, 'gender' => 'F'))
+
+      result = first_users_record
+
+      result['name'].should == 'Test User'
+
+      result['dob'].mday.should == 4
+      result['dob'].mon.should == 7
+      result['dob'].year.should == 1776
+
+      result['gender'].should == 'F'
+
+      result['foreign'].should be_false
+      result['dull'].should be_false
+
+      result['symbolic'].should == :of_course.to_s
+
+      result['interests'].should == 'health,education'
+    end
+
+    it "updates the child record correctly" do
+      squeal_users_document_with_activities(users_document.merge('activities' => [{ :_id => 'a1', 'name' => 'Be expansionist', 'due_date' => as_time(today + 1) }]))
+      result = first_activity_record
+
+      result['name'].should == 'Be expansionist'
+      result['due_date'].mday.should == (today + 1).mday
+      result['due_date'].mon.should == (today + 1).mon
+      result['due_date'].year.should == (today + 1).year
+    end
+  end
+end