couchdb_to_sql 1.0.0

data/exe/couchdb_to_sql

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'couchdb_to_sql'
+
+configuration_file_name = ARGV[0]
+
+unless configuration_file_name
+  puts
+  puts "Syntax: #{$PROGRAM_NAME} <configuration_file.rb>\n"
+  puts 'For the exact syntax of the configuration file, please consult the documentation or the web site: ' \
+       "https://github.com/ecraft/couchdb_to_sql\n\n"
+  exit 1
+end
+
+# Take in the arguments for the configuration file and try to run it
+CouchdbToSql.logger.info "Reading configuration: #{configuration_file_name}"
+
+CouchdbToSql.module_eval(File.open(configuration_file_name).read, configuration_file_name)
+
+# With the configuration loaded, start her up!
+CouchdbToSql.start

data/lib/couchdb_to_sql.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Low level requirements
+require 'active_support/core_ext/object/blank'
+require 'active_support/inflector'
+require 'couchrest'
+require 'httpclient'
+require 'json'
+require 'logging_library'
+require 'set'
+require 'sequel'
+
+# Our stuff
+require 'couchdb_to_sql/changes'
+require 'couchdb_to_sql/schema'
+require 'couchdb_to_sql/document_handler'
+require 'couchdb_to_sql/table_builder'
+require 'couchdb_to_sql/table_deleted_marker'
+require 'couchdb_to_sql/table_destroyer'
+
+module CouchdbToSql
+  extend LoggingLibrary::Loggable
+
+  Error = Class.new(StandardError)
+  InvalidDataError = Class.new(Error)
+
+  COUCHDB_TO_SQL_SEQUENCES_TABLE = :_couchdb_to_sql_sequences
+
+  module_function
+
+  def changes(database, &block)
+    (@changes ||= []) << Changes.new(database, &block)
+  end
+
+  def start
+    threads = []
+    @changes.each do |changes|
+      threads << Thread.new(changes, &:start)
+    end
+    threads.each(&:join)
+  end
+end

data/lib/couchdb_to_sql/changes.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+module CouchdbToSql
+  class Changes
+    COUCHDB_HEARTBEAT  = 30
+    INACTIVITY_TIMEOUT = 70
+    RECONNECT_TIMEOUT  = 15
+
+    attr_reader :source, :schemas, :handlers
+
+    attr_accessor :highest_sequence
+
+    # Start a new Changes instance by connecting to the provided
+    # CouchDB to see if the database exists.
+    def initialize(opts = '', &block)
+      raise 'Block required for changes!' unless block_given?
+
+      @schemas  = {}
+      @handlers = []
+      @source   = CouchRest.database(opts)
+      @http     = HTTPClient.new
+      @http.debug_dev = STDOUT if ENV.key?('DEBUG')
+      @skip_seqs = Set.new
+
+      log_info 'Connected to CouchDB'
+
+      @ember_pouch_mode = false
+      @fail_on_unhandled_document = false
+      @upsert_mode = false
+
+      # Prepare the definitions
+      @dsl_mode = true
+      instance_eval(&block)
+      @dsl_mode = false
+    end
+
+    #### DSL
+
+    # Sets the `ember_pouch_mode` flag. In `ember-pouch` mode, all the data fields are expected to reside within a
+    # `data` node in the document. More information on `ember-pouch` can be found
+    # [here](https://github.com/nolanlawson/ember-pouch).
+    #
+    # @note Dual-purpose method, accepts configuration of setting or returns a previous definition.
+    def ember_pouch_mode
+      if @dsl_mode
+        @ember_pouch_mode ||= true
+      else
+        @ember_pouch_mode
+      end
+    end
+
+    # Sets the `upsert_mode` flag. When running in upsert mode, Sequel's insert_conflict mode is being used. More information
+    # about that can be found
+    # [here](http://sequel.jeremyevans.net/rdoc/files/doc/postgresql_rdoc.html#label-INSERT+ON+CONFLICT+Support)
+    #
+    # @note Dual-purpose method, accepts configuration of setting or returns a previous definition.
+    def upsert_mode
+      if @dsl_mode
+        @upsert_mode ||= true
+      else
+        @upsert_mode
+      end
+    end
+
+    # Sets the "fail on unhandled document" flag, which will turn log errors into runtime exceptions if an unhandled document is
+    # encountered.
+    #
+    # @note Dual-purpose method, accepts configuration of setting or returns a previous definition.
+    def fail_on_unhandled_document
+      if @dsl_mode
+        @fail_on_unhandled_document ||= true
+      else
+        @fail_on_unhandled_document
+      end
+    end
+
+    # @note Dual-purpose method, accepts configuration of database
+    # or returns a previous definition.
+    def database(opts = nil)
+      if opts
+        @database ||= begin
+          Sequel.connect(opts).tap { |conn|
+            next unless ENV.key?('SEQUEL_LOG_LEVEL')
+
+            conn.logger = LoggingLibrary::LoggerFactory.create(self.class.name).tap { |l|
+              l.level = ENV['SEQUEL_LOG_LEVEL'].to_s.downcase.to_sym
+            }
+          }
+        end
+        find_or_create_sequence_number
+      end
+      @database
+    end
+
+    def document(filter = {}, &block)
+      @handlers << DocumentHandler.new(self, filter, &block)
+    end
+
+    def skip_seqs_file(file_path)
+      file_contents = File.read(file_path)
+      seqs = JSON.parse(file_contents)
+      @skip_seqs |= Set.new(seqs)
+    end
+
+    #### END DSL
+
+    def schema(name)
+      @schemas[name.to_sym] ||= Schema.new(database, name)
+    end
+
+    # Start listening to the CouchDB changes feed. By this stage we should have
+    # a sequence id so we know where to start from and all the filters should
+    # have been prepared.
+    def start
+      perform_request
+    end
+
+    def log_debug(message)
+      logger.debug "#{source.name}: #{message}"
+    end
+
+    def log_info(message)
+      logger.info "#{source.name}: #{message}"
+    end
+
+    def log_error(message)
+      logger.error "#{source.name}: #{message}"
+    end
+
+    protected
+
+    def perform_request
+      raise 'Internal error: Highest_sequence is expected to be non-nil' unless highest_sequence
+      log_info "listening to changes feed from sequence number: #{highest_sequence}"
+
+      url = File.join(source.root.to_s, '_changes')
+      uri = URI.parse(url)
+
+      # Authenticate?
+      if uri.user.present? && uri.password.present?
+        @http.set_auth(source.root, uri.user, uri.password)
+      end
+
+      # Make sure the request has the latest sequence
+      query = {
+        feed: 'continuous',
+        heartbeat: COUCHDB_HEARTBEAT * 1000,
+        include_docs: true,
+        since: highest_sequence
+      }
+
+      num_rows = 0
+
+      loop do
+        # Perform the actual request for chunked content
+        @http.get_content(url, query) do |chunk|
+          rows = chunk.split("\n")
+          rows.each { |row|
+            parsed_row = JSON.parse(row)
+            process_row(parsed_row)
+
+            num_rows += 1
+            log_info "Processed #{num_rows} rows" if (num_rows % 10_000) == 0
+          }
+        end
+        log_error "connection ended, attempting to reconnect in #{RECONNECT_TIMEOUT}s..."
+        wait RECONNECT_TIMEOUT
+      end
+    rescue HTTPClient::TimeoutError, HTTPClient::BadResponseError => e
+      log_error "connection failed: #{e.message}, attempting to reconnect in #{RECONNECT_TIMEOUT}s..."
+      wait RECONNECT_TIMEOUT
+      retry
+    end
+
+    def process_row(row)
+      id = row['id']
+      seq = row['seq']
+
+      return if id =~ /^_design/
+      return if @skip_seqs.include?(seq)
+
+      if id
+        # Wrap the whole request in a transaction
+        database.transaction do
+          doc = fetch_document_from(row)
+
+          if row['deleted']
+            log_info "received DELETE seq. #{seq} id: #{id}"
+            handlers.each { |handler| handler.mark_as_deleted(doc) }
+          else
+            log_debug "received CHANGE seq. #{seq} id: #{id}"
+
+            document_handlers = find_document_handlers(doc)
+            if document_handlers.empty?
+              message = 'No document handlers found for document. ' \
+                "Document data: #{doc.inspect}, seq: #{seq}, source: #{@source.name}"
+              raise InvalidDataError, message if fail_on_unhandled_document
+
+              log_error message
+            end
+
+            document_handlers.each do |handler|
+              # Delete all previous entries of doc, then re-create
+              handler.delete(doc)
+              handler.insert(doc)
+            end
+          end
+
+          update_sequence_table(seq)
+        end # transaction
+      elsif row['last_seq']
+        # Sometimes CouchDB will send an update to keep the connection alive
+        log_info "received last seq: #{row['last_seq']}"
+      end
+    end
+
+    def fetch_document_from(row)
+      doc = row.fetch('doc')
+
+      if ember_pouch_mode
+        ember_pouch_transform_document(doc)
+      else
+        doc
+      end
+    end
+
+    def ember_pouch_transform_document(doc)
+      if doc.key?('data')
+        doc['id'] = doc['_id'].split('_2_', 2).last
+        doc.merge(doc.delete('data'))
+      else
+        doc
+      end
+    end
+
+    def find_document_handlers(document)
+      @handlers.select { |row| row.handles?(document) }
+    end
+
+    def find_or_create_sequence_number
+      unless database.table_exists?(CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE)
+        create_sequence_table
+        sequence_table.insert(couchdb_database_name: source.name, created_at: DateTime.now)
+      end
+
+      row = sequence_table.where(couchdb_database_name: source.name).first
+      self.highest_sequence = (row ? row.fetch(:highest_sequence) : '0')
+    end
+
+    def update_sequence_table(new_highest_sequence)
+      if upsert_mode
+        data = {
+          couchdb_database_name: source.name,
+          highest_sequence: new_highest_sequence,
+          updated_at: DateTime.now
+        }
+        sequence_table
+          .insert_conflict(target: :couchdb_database_name, update: data)
+          .insert(data.merge(created_at: data[:updated_at]))
+      else
+        sequence_table
+          .where(couchdb_database_name: source.name)
+          .update(highest_sequence: new_highest_sequence)
+      end
+
+      self.highest_sequence = new_highest_sequence
+    end
+
+    def create_sequence_table
+      database.create_table CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE do
+        String :couchdb_database_name, primary_key: true
+        String :highest_sequence, default: '0', null: false
+        DateTime :created_at
+        DateTime :updated_at
+      end
+    end
+
+    def sequence_table
+      database[CouchdbToSql::COUCHDB_TO_SQL_SEQUENCES_TABLE]
+    end
+
+    def logger
+      CouchdbToSql.logger
+    end
+  end
+end

data/lib/couchdb_to_sql/document_handler.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module CouchdbToSql
+  #
+  # Handles document insertion, deletion and 'marking as deleted' operations.
+  #
+  # This class delegates the actual insertion, deletion etc to the various `Table*` classes.
+  #
+  class DocumentHandler
+    attr_reader :changes, :filter, :mode
+    attr_accessor :document
+
+    def initialize(changes, filter = {}, &block)
+      @changes  = changes
+      @filter   = filter
+      @_block   = block
+      @mode     = nil
+    end
+
+    def handles?(doc)
+      @filter.each do |k, v|
+        return false if doc[k.to_s] != v
+      end
+      true
+    end
+
+    ### START DSL
+
+    # Handle a table definition.
+    def table(name, opts = {}, &block)
+      if @mode == :delete
+        TableDestroyer.new(self, name, opts).execute
+      elsif @mode == :mark_as_deleted
+        TableDeletedMarker.new(self, name, opts).execute
+      elsif @mode == :insert
+        TableBuilder.new(self, name, opts, &block).execute
+      end
+    end
+
+    ### END DSL
+
+    def handler
+      self
+    end
+
+    def primary_keys
+      []
+    end
+
+    def key_filter
+      {}
+    end
+
+    def id
+      document['_id']
+    end
+
+    def rev
+      document['_rev']
+    end
+
+    def insert(document)
+      @mode = :insert
+      self.document = document
+      instance_eval(&@_block)
+    end
+
+    def delete(document)
+      @mode = :delete
+      self.document = document
+      instance_eval(&@_block)
+    end
+
+    def mark_as_deleted(document)
+      @mode = :mark_as_deleted
+      self.document = document
+      instance_eval(&@_block)
+    end
+
+    def schema(name)
+      changes.schema(name)
+    end
+
+    def database
+      changes.database
+    end
+  end
+end

data/lib/couchdb_to_sql/schema.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module CouchdbToSql
+  # Wrapper around a sequel table definition
+  # to allow easy access to column names.
+  class Schema
+    attr_accessor :name, :database, :columns, :column_names
+
+    def initialize(database, name)
+      self.name         = name.to_sym
+      self.database     = database
+      self.columns      = {}
+      self.column_names = []
+      parse_schema
+    end
+
+    def dataset
+      database[name]
+    end
+
+    protected
+
+    def parse_schema
+      database.schema(name).each do |row|
+        column_names << row[0]
+        columns[row[0]] = row[1]
+      end
+    end
+  end
+end