couch_tap 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2667fb312598a6e1960dc06f91b4feec38d2419e
+  data.tar.gz: c45097c89984d980fb67d4996c468cabf28f6fae
+SHA512:
+  metadata.gz: 01ebd6988be11f2ee4d0ca1d4cbbb42d10dcbcd5b7457363da2c11167ab9eae8e468a98c70288517e4477f83b2dd89df3db96405ce2eca0f6a3e871b76c0be59
+  data.tar.gz: e4054c2b4447ffc355e3ca893b708bb7b9ea28a0e4b12da68bd23b628a22d3e3eae3d4728a2b2ac2df276069c271f2cb423be58a66df064ccbd75dd77d243549

data/.gitignore

@@ -0,0 +1,13 @@
+.bundle
+.sass-cache/
+*.sw[opn]
+.rvmrc
+.DS_Store
+*.zip
+*.*~
+.rsync_cache
+zeus.json
+output
+tmp
+crash.log
+pkg

data/Gemfile

@@ -0,0 +1,4 @@
+
+source 'https://rubygems.org'
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,59 @@
+PATH
+  remote: .
+  specs:
+    couch_tap (0.0.2)
+      activesupport (>= 3.0.0)
+      couchrest (~> 1.1.3)
+      em-http-request (~> 1.0.3)
+      sequel (>= 3.45.0)
+      yajl-ruby (~> 1.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.0.2)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    addressable (2.3.6)
+    atomic (1.1.14)
+    cookiejar (0.3.2)
+    couchrest (1.1.3)
+      mime-types (~> 1.15)
+      multi_json (~> 1.0)
+      rest-client (~> 1.6.1)
+    em-http-request (1.0.3)
+      addressable (>= 2.2.3)
+      cookiejar
+      em-socksify
+      eventmachine (>= 1.0.0.beta.4)
+      http_parser.rb (>= 0.5.3)
+    em-socksify (0.3.0)
+      eventmachine (>= 1.0.0.beta.4)
+    eventmachine (1.0.3)
+    http_parser.rb (0.6.0)
+    i18n (0.6.9)
+    metaclass (0.0.1)
+    mime-types (1.25.1)
+    minitest (4.7.5)
+    mocha (0.13.3)
+      metaclass (~> 0.0.1)
+    multi_json (1.9.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    sequel (4.9.0)
+    sqlite3 (1.3.7)
+    thread_safe (0.1.3)
+      atomic
+    tzinfo (0.3.38)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  couch_tap!
+  mocha
+  sqlite3

data/README.md

@@ -0,0 +1,183 @@
+
+# Couch Tap
+
+Utility to listen to a CouchDB changes feed and automatically insert, update,
+or delete rows into a relational database from matching key-value conditions of incoming documents.
+
+While CouchDB is awesome, business people probably won't be
+quite as impressed when they want to play around with the data. Regular SQL
+is generally accepted as being easy to use and much more widely supported by a larger
+range of comercial tools.
+
+Couch Tap will listen to incoming documents on a CouchDB's changes
+stream and automatically update rows of RDBMS tables defined in the
+conversion schema. The changes stream uses a sequence number allowing
+synchronisation to be started and stopped at will.
+
+Ruby's fast and simple (sequel)[http://sequel.jeremyevans.net/] library is used to provide the connection to the
+database. This library can also be used for migrations, important for frequently changing schemas.
+
+Couch tap takes a simple two-step approach converting documents to rows. When a change event is received
+for a matching `document` definition, each associated row is completely deleted. If the change
+is anything other than a delete event, the rows will be re-created with the new data.
+This makes things much easier when trying to deal with multi-level documents (i.e. documents of documents)
+and one-to-many table relationships.
+
+
+## A Couch Tap Project
+
+Couch Tap requires a configuration or filter definition that will allow incoming
+document changes to be identified and dealt with.
+
+The following example attempts to outline most of the key features of the DSL.
+
+```ruby
+# The couchdb database from which to request the changes feed
+changes "http://user:pass@host:port/invoicing" do
+
+  # Which database should we connect to?
+  database "postgres://user:pass@localhost:5432/invoicing"
+
+  # Simple automated copy, each property's value in the matching CouchDB
+  # document will copied to the table field with the same name.
+  document 'type' => 'User' do
+    table :users
+  end
+
+  document 'type' => 'Invoice' do
+
+    table :invoices, :key => :invoice_id do
+
+      # Copy columns from fields with different name
+      column :updated_at, :updated_on
+      column :created_at, :created_on
+
+      # Manually set a value from document or fixed variable
+      column :date, doc['date'].to_json
+      column :added_at, Time.now
+
+      # Set column values from a block.
+      column :total do
+        doc['items'].inject(0){ |sum,item| sum + item['total'] }
+      end
+
+      # Collections perform special synchronization in order to deal with
+      # one to one, or indeed many to many relationships.
+      #
+      # Rather than attempting a complex syncrhonisation process, the current
+      # version of Couch Tap will just DELETE all current entries with a
+      # primary key id that matches that of the parent table.
+      #
+      # The foreign id key is assumed to be name of the parent
+      # table in singular form with `_id` appended.
+      #
+      # Each item provided in the array will be made available in the
+      # `#data` method, and index from `#index`.
+      # `#document` continues to be the complete source document.
+      #
+      # Collections can be nested to create highly complex structures.
+      #
+      collection :groups do
+        table :invoice_groups do
+
+          collection :entries do
+            table :invoice_entries, :key => :entry_id do
+              column :date, data['date']
+              column :updated_at, document['updated_at']
+            end
+          end
+
+        end
+      end
+
+      # Collections can also be used on Many to Many relationships.
+      collection :label_ids do
+        table :invoice_labels do
+          column :label_id, data
+        end
+      end
+
+    end
+
+  end
+end
+```
+
+## DSL Summary
+
+### changes
+
+Defines which CouchDB database should be used to request the changes feed.
+
+After loading the rest of the configuration, the service will
+connect to the database using Event Machine. As new changes come into the
+system, they will be managed in the background.
+
+
+### connection
+
+The Sequel URL used to connect to the destination database. Behind the scenes,
+Couch Tap will check for a table named `couchdb_sequence` that contains a single
+row for the current changes sequence id, much like a migration id typically
+seen in a Rails database.
+
+As changes are received from CouchDB, the current sequence will be updated to
+match.
+
+#### document
+
+When a document is received from the changes feed, it will be passed through each
+`document` stanza looking for a match. Take the following example:
+
+    document :type => 'Invoice' do |doc|
+      # ...
+    end
+
+This will match all documents whose `type` property is equal to "Invoice". The
+document itself will be made available as a hash through the `doc` block variable.
+
+`document` stanzas may be nested if required to provide further levels of
+filtering.
+
+#### table
+
+Each `table` stanza lets Couch Tap know that all or part of the current document
+should be inserted into it. By default, the matching table's schema will be read
+and any field names that match a property in the top-level of the document will
+be inserted automatically.
+
+One of the limitations of Couch Tap is that all tables must have an id field as their
+primary key. In each row, the id's value will be copied from the `_id` of the
+document being imported. This is the only way that deleted documents can be
+reliably found and removed from the relational database.
+
+#### column
+
+#### collection
+
+#### foreign_key
+
+
+### Notes on deleted documents
+
+Synchronising a deleted document is generally a much more complicated operation.
+Given that the original document no longer exists in the CouchDB database,
+there is no way to know which document group and table the document was inserted
+into.
+
+To get around this issue, Couch Tap will search through all the tables defined
+for the database and delete rows that match the primary or foreign keys.
+
+Obviously, this is very inefficient. Fortunately, CouchDB is not really suited
+to systems that require lots of document deletion, so hopefully this won't be
+too much of a problem.
+
+
+## Testing
+
+Run tests using rake, or individual tests as follows:
+
+    rake test TEST=test/unit/changes_test.rb
+
+
+

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+require 'rubygems'
+require 'rake/testtask'
+
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList.new('test/unit/**/*.rb')
+end
+
+desc "Run tests"
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.0.2

data/bin/couch_tap

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'couch_tap'
+
+# Take in the arguments for the configuration file and try to run it
+CouchTap.logger.info "Reading configuration: #{ARGV[0]}"
+
+CouchTap.module_eval(File.open(ARGV[0]).read)
+
+# With the configuration loaded, start her up!
+CouchTap.start
+

data/couch_tap.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name          = "couch_tap"
+  s.version       = `cat VERSION`.strip
+  s.date          = File.mtime('VERSION')
+  s.summary       = "Listen to a CouchDB changes feed and create rows in a relational database in real-time."
+  s.description   = "Couch Tap provides a DSL that allows complex CouchDB documents to be converted into rows in a RDBMS' table. The stream of events received from the CouchDB changes feed will trigger documents to be fed into a matching filter block and saved in the database."
+  s.authors       = ["Sam Lown"]
+  s.email         = 'me@samlown.com'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "couchrest", "~> 1.1.3"
+  s.add_dependency "em-http-request", "~> 1.0.3"
+  s.add_dependency "yajl-ruby", "~> 1.1.0"
+  s.add_dependency "sequel", ">= 3.45.0"
+  s.add_dependency "activesupport", ">= 3.0.0"
+  s.add_development_dependency "mocha"
+  s.add_development_dependency "sqlite3"
+end

data/examples/feed.rb

@@ -0,0 +1,27 @@
+
+# Sample Configuration Script
+#
+# Run using the command line application:
+#
+#     couch_tap feed.rb
+#
+
+
+changes "http://user:pass@host:port/invoicing" do
+
+  # Which database should we connect to?
+  database "sqlite:///database.sqlite3"
+
+  filter 'type' => 'User' do
+    table :users
+  end
+
+  filter 'type' => 'Journey' do
+    table :journeys
+  end
+
+
+end
+
+
+

data/lib/couch_tap.rb

@@ -0,0 +1,48 @@
+
+# Low level requirements
+require 'sequel'
+require 'couchrest'
+require 'em-http'
+require 'yajl'
+require 'logger'
+require 'active_support/inflector'
+require 'active_support/core_ext/object/blank'
+
+# Our stuff
+require 'couch_tap/changes'
+require 'couch_tap/schema'
+require 'couch_tap/document_handler'
+require 'couch_tap/builders/collection'
+require 'couch_tap/builders/table'
+require 'couch_tap/destroyers/collection'
+require 'couch_tap/destroyers/table'
+
+
+module CouchTap
+  extend self
+
+  def changes(database, &block)
+    (@changes ||= []) << Changes.new(database, &block)
+  end
+
+  def start
+    EventMachine.run do
+      @changes.each do |changes|
+        changes.start
+      end
+    end
+  end
+
+  # Provide some way to handle messages
+  def logger
+    @logger ||= prepare_logger
+  end
+
+  def prepare_logger
+    log = Logger.new(STDOUT)
+    log.level = Logger::INFO
+    log
+  end
+
+end
+

data/lib/couch_tap/builders/collection.rb

@@ -0,0 +1,41 @@
+
+module CouchTap
+
+  module Builders
+
+    #
+    # Collection Builder. Go through each sub-table definition and recursively
+    # prepare the data ready to be inserted into the database.
+    #
+    class Collection
+
+      attr_reader :parent, :field
+
+      def initialize(parent, field, opts = {}, &block)
+        @_tables = []
+        @parent  = parent
+        @field   = field
+
+        instance_eval(&block)
+      end
+
+      def execute
+        @_tables.each do |table|
+          table.execute
+        end
+      end
+
+      #### DSL Methods
+
+      def table(name, opts = {}, &block)
+        source = parent.data[field.to_s] || []
+        source.each do |item|
+          options = opts.merge(:data => item)
+          @_tables << Table.new(parent, name, options, &block)
+        end
+      end
+
+    end
+  end
+end
+