squealer 1.0.0

data/.gitignore

@@ -0,0 +1 @@
+pkg/

data/.watchr

@@ -0,0 +1,27 @@
+# ~/.vim/ftdetect/watchr.vim
+#
+# This should have only the following line in it:
+#
+# autocmd BufNewFile,BufRead *.watchr setf ruby
+#
+# This will enable vim to recognize this file as ruby code should you wish to
+# edit it.
+def run(cmd)
+  puts cmd
+  system cmd
+end
+
+def spec(file)
+  run "spec -O spec/spec.opts #{file}"
+end
+
+watch("spec/.*/*_spec\.rb") do |match|
+  p match[0]
+  spec(match[0])
+end
+
+watch("lib/(.*/.*)\.rb") do |match|
+  p match[1]
+  spec("spec/#{match[1]}_spec.rb")
+end
+

data/README.md

@@ -0,0 +1,13 @@
+# Squealer
+
+For now, this is specifically for MongoDB exporting to mySQL with the assumption that the data will be heavily denormalized.
+
+The target SQL database must have no foreign keys.
+
+The target SQL database must use a primary key of char(16) with value of the MongoDB id.
+
+It is assumed no indexes are present in the target database table.
+
+The target row is inserted, or updated if present. We are using MySQL INSERT ... UPDATE ON DUPLICATE KEY extended syntax to achieve this for now. This allows an event-driven update of exported data as well as a bulk batch process.
+
+

data/Rakefile

@@ -0,0 +1,27 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require 'rake'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+task :default => [:spec]
+Rake::Task[:default].clear
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "squealer"
+    gemspec.summary = "Document-oriented to Relational database exporter"
+    gemspec.description = "Exports mongodb to mysql. More later."
+    gemspec.email = "joshua.graham@grahamis.com"
+    gemspec.homepage = "http://github.com/delitescere/squealer/"
+    gemspec.authors = ["Josh Graham", "Durran Jordan"]
+    gemspec.add_dependency('mysql', '>= 2.8.1')
+    gemspec.add_dependency('mongo', '>= 0.18.3')
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/example_squeal.rb

@@ -0,0 +1,38 @@
+require 'squealer'
+
+import('localhost', 27017, 'development')
+export('localhost', 'root', '', 'reporting_export')
+
+import.collection("users").find({}).each do |user|
+  target(:user, user._id) do  # insert or update on user where id is primary key column name
+    assign(:name) { user.first_name + " " + user.last_name.upcase }
+    assign(:dob) { user.dob }
+
+    user.activities.each do |activity|
+      target(:activity, activity._id) do
+        assign(:user_id) { user._id }
+        assign(:name) { activity.name }
+      end
+
+      activity.tasks.each do |task|
+        target(:task, task._id) do
+          assign(:user_id) { user._id }
+          assign(:activity_id) { activity._id }
+          assign(:date) { task.date }
+        end
+      end #activity.tasks
+    end #user.activities
+  end
+end #collection("users")
+
+import.collection("organization").find({}).each do |organization|
+  if organization.disabled
+    import.collection("users").find({ :organization_id => organization.id }) do |user|
+      target(:user, user.id) do
+        assign(:disabled) { true }
+      end
+    end
+  else
+    # something else
+  end
+end

data/lib/squealer/database.rb

@@ -0,0 +1,26 @@
+require 'mysql'
+require 'mongo'
+require 'singleton'
+
+module Squealer
+  class Database
+    include Singleton
+
+    def import_from(host, port, name)
+      @import_dbc = Mongo::Connection.new(host, port, :slave_ok => true).db(name)
+    end
+
+    def export_to(host, username, password, name)
+      @export_dbc = Mysql.connect(host, username, password, name)
+    end
+
+    def import
+      @import_dbc
+    end
+
+    def export
+      @export_dbc
+    end
+
+  end
+end

data/lib/squealer/hash.rb

@@ -0,0 +1,6 @@
+class Hash
+  def method_missing(name, *args, &block)
+    super if args.size > 0 || block_given?
+    self[name.to_s]
+  end
+end

data/lib/squealer/object.rb

@@ -0,0 +1,33 @@
+class Object
+
+  def target(table_name, row_id, &block)
+    Squealer::Target.new(Squealer::Database.instance.export, table_name, row_id, &block)
+  end
+
+  def assign(column_name, &block)
+    Squealer::Target.current.assign(column_name, &block)
+  end
+
+  def import(*args)
+    if args.length > 0
+      Squealer::Database.instance.import_from(*args)
+    else
+      Squealer::Database.instance.import
+    end
+  end
+
+  def export(*args)
+    if args.length > 0
+      Squealer::Database.instance.export_to(*args)
+    else
+      Squealer::Database.instance.export
+    end
+  end
+
+end
+
+class NilClass
+  def each
+    []
+  end
+end

data/lib/squealer/target.rb

@@ -0,0 +1,106 @@
+require 'delegate'
+require 'singleton'
+
+module Squealer
+  class Target
+
+    def self.current
+      Queue.instance.current
+    end
+
+    def initialize(database_connection, table_name, row_id, &block)
+      throw "Block must be given to target (otherwise, there's no work to do)" unless block_given?
+
+      @table_name = table_name.to_s
+      @row_id = row_id
+      @column_names = []
+      @column_values = []
+      @sql = ''
+
+      target(&block)
+    end
+
+    def sql
+      @sql
+    end
+
+    def assign(column_name, &block)
+      @column_names << column_name
+      @column_values << block.call
+    end
+
+
+    private
+
+    def target(&block)
+      Queue.instance.push(self)
+
+      yield self
+
+      @sql = "INSERT #{@table_name}"
+      @sql << " (#{pk_name}#{column_names}) VALUES (?#{column_value_markers})"
+      @sql << " ON DUPLICATE KEY UPDATE #{column_markers}"
+
+      execute_sql(@sql)
+
+      Queue.instance.pop
+    end
+
+    def self.targets
+      @@targets
+    end
+
+    def targets
+      @@targets
+    end
+
+    def execute_sql(sql)
+      statement = Database.instance.export.prepare(sql)
+      values = [*column_values] + [*column_values]  #array expando
+      statement.send(:execute, @row_id, *values) #expand values into distinct arguments
+    end
+
+    def pk_name
+      'id'
+    end
+
+    def column_names
+      return if @column_names.size == 0
+      ",#{@column_names.join(',')}"
+    end
+
+    def column_values
+      @column_values
+    end
+
+    def column_value_markers
+      return if @column_names.size == 0
+      result = ""
+      @column_names.size.times { result << ',?'}
+      result
+    end
+
+    def column_markers
+      return if @column_names.size == 0
+      result = ""
+      @column_names.each {|k| result << "#{k}=?," }
+      result.chop
+    end
+
+    class Queue < DelegateClass(Array)
+      include Singleton
+
+      def current
+        last
+      end
+
+      protected
+
+      def initialize
+        super([])
+      end
+
+    end
+
+  end
+end

data/lib/squealer/time.rb

@@ -0,0 +1,5 @@
+class Time
+  def to_s
+    self.strftime("%Y-%m-%d %H:%M:%S %Z")
+  end
+end

data/lib/squealer.rb

@@ -0,0 +1,6 @@
+require 'squealer/object'
+require 'squealer/hash'
+require 'squealer/time'
+
+require 'squealer/target'
+require 'squealer/database'

data/lib/tasks/jeweler.rake

@@ -0,0 +1,15 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "squealer"
+    gemspec.summary = "Document-oriented to Relational database exporter"
+    gemspec.description = "Exports mongodb to mysql. More later."
+    gemspec.email = "joshua.graham@grahamis.com"
+    gemspec.homepage = "http://github.com/deltiscere/squealer/"
+    gemspec.authors = ["Josh Graham", "Durran Jordan"]
+    gem.add_dependency('mysql', '>= 2.8.1')
+    gem.add_dependency('mongo', '>= 0.18.3')
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/spec/spec.opts

@@ -0,0 +1,3 @@
+--colour
+--format nested
+--drb

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'spork'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'squealer'

data/spec/squealer/database_spec.rb

@@ -0,0 +1,41 @@
+require 'mysql'
+require 'mongo'
+require 'spec_helper'
+
+describe Squealer::Database do
+
+  before(:all) do
+    @db_name = "test_export_#{object_id}"
+    create_test_db(@db_name)
+  end
+
+  after(:all) do
+    drop_test_db(@db_name)
+  end
+
+  it "is a singleton" do
+    Squealer::Database.respond_to?(:instance).should be_true
+  end
+
+  it "takes an import database" do
+    Squealer::Database.instance.import_from('localhost', 27017, @db_name)
+    Squealer::Database.instance.import.should be_a_kind_of(Mongo::DB)
+  end
+
+  it "takes an export database" do
+    Squealer::Database.instance.export_to('localhost', 'root', '', @db_name)
+    Squealer::Database.instance.export.should be_a_kind_of(Mysql)
+  end
+
+  private
+
+  def create_test_db(name)
+    @my = Mysql.connect('localhost', 'root')
+    @my.query("create database #{name}")
+  end
+
+  def drop_test_db(name)
+    @my.query("drop database #{name}")
+    @my.close
+  end
+end

data/spec/squealer/hash_spec.rb

@@ -0,0 +1,28 @@
+require "spec_helper"
+
+describe Hash do
+
+  describe "#method_missing" do
+
+    it "treats it as a hash key lookup" do
+      { "name" => "Josh" }.name.should == "Josh"
+    end
+
+    context "with args" do
+
+      it "treats it normally" do
+        lambda { { "name" => "Josh" }.name(nil) }.should raise_error(NoMethodError)
+      end
+
+    end
+
+    context "with block" do
+
+      it "treats it normally" do
+        lambda { { "name" => "Josh" }.name {nil} }.should raise_error(NoMethodError)
+      end
+
+    end
+
+  end
+end

data/spec/squealer/object_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+
+describe NilClass do
+
+  describe "#each" do
+    it "returns an empty array" do
+      nil.each.should == [] # because mongo is schema-less
+    end
+  end
+end
+
+describe Object do
+
+  describe "#target" do
+
+    it "has been defined" do
+      Object.new.respond_to?(:target).should be_true
+    end
+
+    it "invokes Squealer::Target.new" do
+      Squealer::Target.should_receive(:new)
+      target(:test_table, 1) { nil }
+    end
+
+    it "uses the export database connection" do
+      mock_mysql
+      target(:test_table, 1) { nil }
+    end
+
+  end
+
+  describe "#assign" do
+
+    it "has been defined" do
+      Object.new.respond_to?(:assign).should be_true
+    end
+
+    it "invokes assign on the target it is immediately nested within" do
+      mock_mysql
+      target(:test_table, 1) do |target1|
+        target1.should_receive(:assign)
+        assign(:colA) { 42 }
+
+        target(:test_table_2, 1) do |target2|
+          target2.should_receive(:assign)
+          assign(:colspeak) { 1984 }
+        end
+      end
+    end
+
+  end
+
+  describe "#import" do
+
+    it "has been defined" do
+      Object.new.respond_to?(:import).should be_true
+    end
+
+    context "with a single argument" do
+
+      it "invokes Database.import_from" do
+        Squealer::Database.instance.should_receive(:import_from)
+        import('test_import')
+      end
+
+    end
+
+    context "with no argment" do
+
+      it "invokes Database.import" do
+        Squealer::Database.instance.should_receive(:import)
+        import
+      end
+
+    end
+
+  end
+
+  describe "#export" do
+
+    it "has been defined" do
+      Object.new.respond_to?(:export).should be_true
+    end
+
+    context "with a single argument" do
+
+      it "invokes Database.export_to" do
+        Squealer::Database.instance.should_receive(:export_to)
+        export('test_export')
+      end
+
+    end
+
+    context "with no argment" do
+
+      it "invokes Database.export" do
+        Squealer::Database.instance.should_receive(:export)
+        export
+      end
+
+    end
+
+  end
+
+  def mock_mysql
+    my = mock(Mysql)
+    st = mock(Mysql::Stmt)
+    Squealer::Database.instance.should_receive(:export).at_least(:once).and_return(my)
+    my.should_receive(:prepare).at_least(:once).and_return(st)
+    st.should_receive(:execute).at_least(:once)
+  end
+
+end

data/spec/squealer/target_spec.rb

@@ -0,0 +1,135 @@
+require 'spec_helper'
+
+describe Squealer::Target do
+  let(:export_dbc) { mock(Mysql) }
+  let(:table_name) { :test_table }
+  let(:row_id) { 0 }
+
+  before(:each) do
+    Squealer::Database.instance.should_receive(:export).at_least(:once).and_return(export_dbc)
+    st = mock(Mysql::Stmt)
+    export_dbc.should_receive(:prepare).at_least(:once).and_return(st)
+    st.should_receive(:execute).at_least(:once)
+  end
+
+  it "sends the sql to the export database" do
+    Squealer::Target.new(export_dbc, table_name, row_id) { nil }
+  end
+
+  describe "#target" do
+
+    it "pushes itself onto the targets stack when starting" do
+      @target1 = nil
+      @target2 = nil
+      target1 = Squealer::Target.new(export_dbc, table_name, row_id) do
+        @target1 = Squealer::Target.current
+        Squealer::Target.new(export_dbc, "#{table_name}_2", row_id) do
+          @target2 = Squealer::Target.current
+          @target2.should_not == @target1
+        end
+      end
+      target1.should === @target1
+    end
+
+    it "pops itself off the targets stack when finished" do
+      Squealer::Target.new(export_dbc, table_name, row_id) { nil }
+      Squealer::Target.current.should be_nil
+    end
+
+    context "generates SQL command strings" do
+
+      let(:target) { Squealer::Target.new(export_dbc, table_name, row_id) { nil } }
+
+      it "targets the table" do
+        target.sql.should =~ /^INSERT #{table_name} /
+      end
+
+      it "uses an INSERT ... ON DUPLICATE KEY UPDATE statement" do
+        target.sql.should =~ /^INSERT .* ON DUPLICATE KEY UPDATE /
+      end
+
+      it "includes the primary key name in the INSERT" do
+        target.sql.should =~ / \(id\) VALUES/
+      end
+
+      it "includes the primary key value in the INSERT" do
+        # target.sql.should =~ / VALUES \('#{row_id}'\) /
+        target.sql.should =~ / VALUES \(\?\) /
+      end
+
+    end
+
+    context "with inner block" do
+
+      it "yields inner blocks" do
+        block_done = false
+        target = Squealer::Target.new(export_dbc, table_name, row_id) { block_done = true }
+        block_done.should be_true
+      end
+
+      it "yields inner blocks first" do
+        Squealer::Target.new(export_dbc, table_name, row_id) { Squealer::Target.current.sql.should be_empty }
+      end
+
+      it "yields inner blocks first and they can assign to this target" do
+        target = Squealer::Target.new(export_dbc, table_name, row_id) { Squealer::Target.current.assign(:colA) { 42 } }
+        target.sql.should =~ /colA/
+        # target.sql.should =~ /42/
+        target.sql.should =~ /\?/
+      end
+
+      context "with 2 columns" do
+
+        let(:value_1) { 42 }
+        let(:target) do
+          Squealer::Target.new(export_dbc, table_name, row_id) { Squealer::Target.current.assign(:colA) { value_1 } }
+        end
+
+        it "includes the column name in the INSERT" do
+          target.sql.should =~ /\(id,colA\) VALUES/
+        end
+
+        it "includes the column value in the INSERT" do
+          # target.sql.should =~ /VALUES \('#{row_id}','#{value_1}'\)/
+          target.sql.should =~ /VALUES \(\?,\?\)/
+        end
+
+        it "includes the column name and value in the UPDATE" do
+          # target.sql.should =~ /UPDATE colA='#{value_1}'/
+          target.sql.should =~ /UPDATE colA=\?/
+        end
+
+      end
+
+      context "with 3 columns" do
+
+        let(:value_1) { 42 }
+        let(:value_2) { 'foobar' }
+        let(:target) do
+          Squealer::Target.new(export_dbc, table_name, row_id) do
+            Squealer::Target.current.assign(:colA) { value_1 }
+            Squealer::Target.current.assign(:colB) { value_2 }
+          end
+        end
+
+        it "includes the column names in the INSERT" do
+          target.sql.should =~ /\(id,colA,colB\) VALUES/
+        end
+
+        it "includes the column values in the INSERT" do
+          # target.sql.should =~ /VALUES \('#{row_id}','#{value_1}','#{value_2}'\)/
+          target.sql.should =~ /VALUES \(\?,\?,\?\)/
+        end
+
+        it "includes the column names and values in the UPDATE" do
+          # target.sql.should =~ /UPDATE colA='#{value_1}',colB='#{value_2}'/
+          target.sql.should =~ /UPDATE colA=\?,colB=\?/
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/spec/squealer/time_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe Time do
+
+  describe "#to_s" do
+    it "uses a MySQL compliant format" do
+      Time.gm(2000,"Jan",31).to_s.should == "2000-01-31 00:00:00 UTC"
+    end
+  end
+end

data/squealer.gemspec

@@ -0,0 +1,71 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{squealer}
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Josh Graham", "Durran Jordan"]
+  s.date = %q{2010-03-23}
+  s.description = %q{Exports mongodb to mysql. More later.}
+  s.email = %q{joshua.graham@grahamis.com}
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    ".gitignore",
+     ".watchr",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "lib/example_squeal.rb",
+     "lib/squealer.rb",
+     "lib/squealer/database.rb",
+     "lib/squealer/hash.rb",
+     "lib/squealer/object.rb",
+     "lib/squealer/target.rb",
+     "lib/squealer/time.rb",
+     "lib/tasks/jeweler.rake",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "spec/squealer/database_spec.rb",
+     "spec/squealer/hash_spec.rb",
+     "spec/squealer/object_spec.rb",
+     "spec/squealer/target_spec.rb",
+     "spec/squealer/time_spec.rb",
+     "squealer.gemspec"
+  ]
+  s.homepage = %q{http://github.com/delitescere/squealer/}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Document-oriented to Relational database exporter}
+  s.test_files = [
+    "spec/spec_helper.rb",
+     "spec/squealer/database_spec.rb",
+     "spec/squealer/hash_spec.rb",
+     "spec/squealer/object_spec.rb",
+     "spec/squealer/target_spec.rb",
+     "spec/squealer/time_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<mysql>, [">= 2.8.1"])
+      s.add_runtime_dependency(%q<mongo>, [">= 0.18.3"])
+    else
+      s.add_dependency(%q<mysql>, [">= 2.8.1"])
+      s.add_dependency(%q<mongo>, [">= 0.18.3"])
+    end
+  else
+    s.add_dependency(%q<mysql>, [">= 2.8.1"])
+    s.add_dependency(%q<mongo>, [">= 0.18.3"])
+  end
+end
+

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification 
+name: squealer
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Josh Graham
+- Durran Jordan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-23 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mysql
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 8
+        - 1
+        version: 2.8.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 18
+        - 3
+        version: 0.18.3
+  type: :runtime
+  version_requirements: *id002
+description: Exports mongodb to mysql. More later.
+email: joshua.graham@grahamis.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- .gitignore
+- .watchr
+- README.md
+- Rakefile
+- VERSION
+- lib/example_squeal.rb
+- lib/squealer.rb
+- lib/squealer/database.rb
+- lib/squealer/hash.rb
+- lib/squealer/object.rb
+- lib/squealer/target.rb
+- lib/squealer/time.rb
+- lib/tasks/jeweler.rake
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/squealer/database_spec.rb
+- spec/squealer/hash_spec.rb
+- spec/squealer/object_spec.rb
+- spec/squealer/target_spec.rb
+- spec/squealer/time_spec.rb
+- squealer.gemspec
+has_rdoc: true
+homepage: http://github.com/delitescere/squealer/
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Document-oriented to Relational database exporter
+test_files: 
+- spec/spec_helper.rb
+- spec/squealer/database_spec.rb
+- spec/squealer/hash_spec.rb
+- spec/squealer/object_spec.rb
+- spec/squealer/target_spec.rb
+- spec/squealer/time_spec.rb