couchdb_to_sql 1.0.0

data/lib/couchdb_to_sql/table_builder.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'couchdb_to_sql/table_operator'
+
+module CouchdbToSql
+  #
+  # Table definition handler which handles database INSERT operations.
+  #
+  class TableBuilder < TableOperator
+    attr_reader :attributes
+    attr_reader :data
+
+    def initialize(parent, table_name, opts = {}, &block)
+      @_collections = []
+
+      @parent     = parent
+      @data       = opts[:data] || parent.document
+      @table_name = table_name.to_sym
+
+      deduce_primary_key(opts)
+
+      @attributes = {}
+      set_primary_key_attribute
+      set_attributes_from_data
+
+      instance_eval(&block) if block_given?
+    end
+
+    def id
+      handler.id
+    end
+
+    def document
+      parent.document
+    end
+    alias doc document
+
+    def database
+      @database ||= handler.database
+    end
+
+    #### DSL Methods
+
+    def column(*args)
+      column = args.first
+      field  = args.last
+
+      if block_given?
+        set_attribute(column, yield)
+      elsif field.is_a?(Symbol)
+        set_attribute(column, data[field.to_s])
+      elsif args.length > 1
+        set_attribute(column, field)
+      end
+    end
+
+    #### Support Methods
+
+    def execute
+      # Insert the record and prepare ID for sub-tables
+      id = dataset.insert(attributes)
+      set_attribute(primary_key, id) unless id.blank?
+    end
+
+    private
+
+    def schema
+      handler.schema(table_name)
+    end
+
+    def dataset
+      database[table_name]
+    end
+
+    # Set the primary key in the attributes so that the insert request will have all it requires.
+    def set_primary_key_attribute
+      base = {}
+      base[primary_key] = id
+
+      attributes.update(base)
+    end
+
+    # Take the document and try to automatically set the fields from the columns
+    def set_attributes_from_data
+      return unless data.is_a?(Hash) || data.is_a?(CouchRest::Document)
+      data.each do |k, v|
+        k = k.to_sym
+        next if %i[_id _rev].include?(k)
+        set_attribute(k, v) if schema.column_names.include?(k)
+      end
+    end
+
+    def set_attribute(name, value)
+      name   = name.to_sym
+      column = schema.columns[name]
+      return if column.nil?
+
+      # Perform basic typecasting to avoid errors with empty fields in databases that do not support them.
+      case column[:type]
+      when :string
+        value = value.nil? ? nil : value.to_s
+      when :integer
+        value = value.to_i
+      when :float
+        value = value.to_f
+      else
+        value = nil if value.to_s.empty?
+      end
+      attributes[name] = value
+    end
+  end
+end

data/lib/couchdb_to_sql/table_deleted_marker.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'couchdb_to_sql/table_operator'
+
+module CouchdbToSql
+  #
+  # Table definition handler for handling database UPSERT operations.
+  #
+  class TableDeletedMarker < TableBuilder
+    def execute
+      dataset = handler.database[table_name]
+
+      if attributes.key?(:id)
+        handler.changes.log_info "Deletion with 'id' field present (#{primary_key} '#{handler.id}'), assuming tombstone. " \
+                                 'Updating data in SQL/Postgres database with data from CouchDB document.'
+        fields = attributes.merge(
+          _deleted: true,
+          _deleted_timestamp: Sequel::CURRENT_TIMESTAMP,
+          rev: handler.rev
+        )
+
+        dataset
+          .insert_conflict(
+            target: primary_key,
+            update: fields
+          )
+          .insert(fields)
+      else
+        handler.changes.log_info "Found deletion without 'id' field (#{primary_key} '#{handler.id}'), assuming plaque. Leaving " \
+                                 'data as-is in SQL/Postgres, only setting _deleted* fields.'
+        records_modified = dataset
+          .where(key_filter)
+          .update(
+            _deleted: true,
+            _deleted_timestamp: Sequel::CURRENT_TIMESTAMP,
+            rev: handler.rev
+          )
+
+        handler.changes.log_info "#{records_modified} record(s) were marked as deleted"
+      end
+    end
+
+    def key_filter
+      {
+        primary_key => handler.id
+      }
+    end
+  end
+end

data/lib/couchdb_to_sql/table_destroyer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'couchdb_to_sql/table_operator'
+
+module CouchdbToSql
+  #
+  # The table destroyer will go through a table definition and make sure that
+  # all rows that belong to the document's id are deleted from the system.
+  #
+  class TableDestroyer < TableOperator
+    def execute
+      dataset = handler.database[table_name]
+      dataset.where(key_filter).delete
+    end
+
+    def key_filter
+      {
+        primary_key => handler.id
+      }
+    end
+  end
+end

data/lib/couchdb_to_sql/table_operator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module CouchdbToSql
+  #
+  # Abstract base class for classes which performs table operations (build, destroy, upsert, etc.)
+  #
+  class TableOperator
+    # @return [DocumentHandler]
+    attr_reader :parent
+
+    # @return [String]
+    attr_reader :table_name
+
+    # @return [Symbol]
+    attr_reader :primary_key
+
+    def initialize(parent, table_name, opts = {})
+      @parent = parent
+      @table_name = table_name
+
+      deduce_primary_key(opts)
+    end
+
+    def deduce_primary_key(opts)
+      @primary_key = (opts[:primary_key] || "#{@table_name.to_s.singularize}_id").to_sym
+    end
+
+    def handler
+      parent.handler
+    end
+
+    def execute
+      raise NotImplementedError, "Classes deriving from #{self} must implement the 'execute' method."
+    end
+  end
+end

data/test/functional/functional_changes_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require '../test_helper'
+
+class FunctionalChangesTest < Test::Unit::TestCase
+  def setup
+    # Create a new CouchDB
+    @source = CouchRest.database('couchdb_to_sql')
+    create_sample_documents
+
+    # Create a new Sqlite DB in memory
+    @database = Sequel.sqlite
+    migrate_sample_database
+  end
+
+  def test_something
+    assert_equal 'foo', 'bar'
+  end
+
+  protected
+
+  def migrate_sample_database
+    @database.create_table :items do
+      primary_key :id
+      String :name
+      Float :price
+      Time :created_at
+    end
+  end
+
+  def create_sample_documents
+    @source.save_doc(name: 'Item 1', price: 1.23, created_at: Time.now)
+    @source.save_doc(name: 'Item 2', price: 2.23, created_at: Time.now)
+    @source.save_doc(name: 'Item 3', price: 3.23, created_at: Time.now)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,30 @@
+
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'simplecov'
+
+SimpleCov.start do
+  add_filter 'test/'
+end
+
+require 'test/unit'
+require 'mocha/setup'
+require 'couchdb_to_sql'
+
+TEST_COUCHDB_URL_PREFIX = ENV.fetch('COUCHDB_URL', 'http://127.0.0.1:5984/').freeze
+TEST_COUCHDB_NAME = 'couchdb_to_sql'
+TEST_COUCHDB_URL = File.join(TEST_COUCHDB_URL_PREFIX, TEST_COUCHDB_NAME)
+TEST_COUCHDB = CouchRest.database(TEST_COUCHDB_URL)
+
+def reset_test_couchdb!
+  TEST_COUCHDB.recreate!
+end
+
+def reset_test_sql_db!(connection_string)
+  Sequel.connect(connection_string) do |db|
+    db.drop_table?(CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE)
+  end
+end

data/test/unit/changes_test.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class ChangesTest < Test::Unit::TestCase
+  def setup
+    reset_test_couchdb!
+    reset_test_sql_db!(sql_connection_string)
+    build_sample_config
+  end
+
+  def test_basic_init
+    @database = @changes.database
+    assert @changes.database, 'Did not assign a database'
+    assert @changes.database.is_a?(Sequel::Database)
+    row = @database[CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE].first
+    assert row, "Did not create a #{CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE} table"
+    assert_equal row.fetch(:highest_sequence), '0', 'Did not set a default sequence number'
+    assert_equal row.fetch(:couchdb_database_name), TEST_COUCHDB_NAME
+  end
+
+  def test_defining_document_handler
+    assert_equal @changes.handlers.length, 3
+    handler = @changes.handlers.first
+    assert handler.is_a?(CouchdbToSql::DocumentHandler)
+    assert_equal handler.filter, type: 'Foo'
+  end
+
+  def test_inserting_rows
+    doc = {
+      '_id' => '1234',
+      'type' => 'Foo',
+      'name' => 'Some Document'
+    }
+    row = {
+      'seq' => 1,
+      'id' => '1234',
+      'doc' => doc
+    }
+
+    handler = @changes.handlers.first
+    handler.expects(:delete).with(doc)
+    handler.expects(:insert).with(doc)
+
+    @changes.send(:process_row, row)
+
+    # Should update seq
+    assert_equal @changes.database[CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE].first[:highest_sequence], '1'
+  end
+
+  def test_inserting_rows_with_multiple_filters
+    row = {
+      'seq' => 3,
+      'id' => '1234',
+      'doc' => {
+        'type' => 'Bar',
+        'special' => true,
+        'name' => 'Some Document'
+      }
+    }
+
+    handler = @changes.handlers[0]
+    handler.expects(:insert).never
+
+    handler = @changes.handlers[1]
+    handler.expects(:delete)
+    handler.expects(:insert)
+
+    handler = @changes.handlers[2]
+    handler.expects(:delete)
+    handler.expects(:insert)
+
+    @changes.send(:process_row, row)
+    assert_equal @changes.database[CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE].first.fetch(:highest_sequence), '3'
+  end
+
+  def test_deleting_rows
+    doc = {
+      'order_number' => '12345',
+      'customer_number' => '54321'
+    }
+    row = {
+      'seq' => 9,
+      'id' => '1234',
+      'deleted' => true,
+      'doc' => doc
+    }
+
+    @changes.handlers.each do |handler|
+      handler.expects(:mark_as_deleted).with(doc)
+    end
+
+    @changes.send(:process_row, row)
+
+    assert_equal @changes.database[CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE].first.fetch(:highest_sequence), '9'
+  end
+
+  def test_returning_schema
+    schema = mock
+    CouchdbToSql::Schema.expects(:new).once.with(@changes.database, :items).returns(schema)
+
+    # Run twice to ensure cached
+    assert_equal @changes.schema(:items), schema
+    assert_equal @changes.schema(:items), schema
+  end
+
+  protected
+
+  def build_sample_config
+    connection_string = sql_connection_string
+
+    @changes = CouchdbToSql::Changes.new(TEST_COUCHDB_URL) do
+      database connection_string
+
+      document type: 'Foo' do
+      end
+
+      document type: 'Bar' do
+      end
+
+      document type: 'Bar', special: true do
+      end
+    end
+  end
+
+  def sql_connection_string
+    ENV.fetch('TEST_SQL_URL', 'sqlite:/')
+  end
+end

data/test/unit/document_handler_test.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class DocumentHandlerTest < Test::Unit::TestCase
+  def test_init
+    @handler = CouchdbToSql::DocumentHandler.new 'changes' do
+      # nothing
+    end
+    assert_equal @handler.changes, 'changes'
+  end
+
+  def test_handles_with_basic_hash
+    @handler = CouchdbToSql::DocumentHandler.new 'changes', type: 'Item'
+    doc = { 'type' => 'Item', '_id' => '1234' }
+    assert @handler.handles?(doc)
+    doc = { 'type' => 'Client', '_id' => '1234' }
+    assert !@handler.handles?(doc)
+  end
+
+  def test_handles_with_multi_level_hash
+    @handler = CouchdbToSql::DocumentHandler.new 'changes', type: 'Item', foo: 'bar'
+    doc = { 'type' => 'Item', 'foo' => 'bar', '_id' => '1234' }
+    assert @handler.handles?(doc)
+    doc = { 'type' => 'Item', '_id' => '1234' }
+    assert !@handler.handles?(doc)
+    doc = { 'foor' => 'bar', '_id' => '1234' }
+    assert !@handler.handles?(doc)
+  end
+
+  def test_id
+    @handler = CouchdbToSql::DocumentHandler.new 'changes' do
+      table :items
+    end
+    @handler.document = { '_id' => '12345' }
+    assert_equal @handler.id, '12345'
+  end
+
+  def test_insert
+    @handler = CouchdbToSql::DocumentHandler.new 'changes' do
+      table :items
+    end
+    @handler.expects(:table).with(:items)
+    doc = { 'type' => 'Foo', '_id' => '1234' }
+    @handler.insert(doc)
+    assert_equal @handler.document, doc
+  end
+
+  def test_delete
+    @handler = CouchdbToSql::DocumentHandler.new 'changes' do
+      table :items
+    end
+    @handler.expects(:table).with(:items)
+    @handler.delete('_id' => '1234')
+    assert_equal @handler.id, '1234'
+  end
+
+  def test_table_definition_on_delete
+    @handler = CouchdbToSql::DocumentHandler.new 'changes' do
+      @mode = :delete # Force delete mode!
+    end
+    @handler.instance_eval('@mode = :delete')
+    @table = mock
+    @table.expects(:execute)
+    CouchdbToSql::TableDestroyer.expects(:new).with(@handler, :items, {}).returns(@table)
+    @handler.table(:items)
+  end
+
+  def test_table_definition_on_insert
+    @handler = CouchdbToSql::DocumentHandler.new 'changes' do
+      # Force insert mode!
+    end
+    @handler.instance_eval('@mode = :insert')
+    @table = mock
+    @table.expects(:execute)
+    CouchdbToSql::TableBuilder.expects(:new).with(@handler, :items, {}).returns(@table)
+    @handler.table(:items)
+  end
+end