couchdb_to_sql 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5a78893ba22f84ef03fa138cbee46b65ab1e793a
+  data.tar.gz: a57a1d876ff3969ece6ddfca90ba32c8d0d56216
+SHA512:
+  metadata.gz: 04bf0493beadf5d5265df76e5778fccb65f46d571b0b45fa2910f33b6b3eff69261f230f78715aa3dfb88c1980e8f034e1c7448a228b798b057a587b13741753
+  data.tar.gz: ce176aaf1253318c74a97ba6cef7bedf1c62d219b5c49301091f84f6cc796f6e2951315375594836bfbc247913d4c455c3b194ecb8a835fdd1529cff5a70931b

data/.gitignore

@@ -0,0 +1,16 @@
+.bundle
+.sass-cache/
+*.sw[opn]
+.rvmrc
+.DS_Store
+*.zip
+*.*~
+.rsync_cache
+zeus.json
+output
+tmp
+crash.log
+pkg
+Gemfile.lock
+bin/
+coverage/

data/.rubocop.yml

@@ -0,0 +1,33 @@
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 2.3
+  DisplayCopNames: true
+  Exclude:
+    - bin/**
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+Layout/MultilineOperationIndentation:
+  EnforcedStyle: indented
+
+Lint/EndAlignment:
+  EnforcedStyleAlignWith: variable
+
+Metrics/LineLength:
+  Max: 132
+Metrics/MethodLength:
+  Severity: warning
+
+Naming/FileName:
+  Enabled: false
+
+# Rationale: allow Weirich-style blocks
+Style/BlockDelimiters:
+  Enabled: false
+Style/Documentation:
+  Enabled: false
+Style/Encoding:
+  Enabled: false
+Style/NumericPredicate:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,39 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2017-10-02 12:02:20 +0300 using RuboCop version 0.50.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 8
+Metrics/AbcSize:
+  Max: 36
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/BlockLength:
+  Max: 59
+
+# Offense count: 1
+# Configuration parameters: CountBlocks.
+Metrics/BlockNesting:
+  Max: 4
+
+# Offense count: 2
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 202
+
+# Offense count: 2
+Metrics/CyclomaticComplexity:
+  Max: 8
+
+# Offense count: 11
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 31
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 10

data/.ruby-version

@@ -0,0 +1 @@
+2.4.2

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+sudo: false
+before_install:
+  - which bundle || gem install bundler
+
+rvm:
+  - 2.3.4
+  - 2.4.2
+  - jruby-9.1.13.0
+
+services:
+  - couchdb

data/.vscode/launch.json

@@ -0,0 +1,46 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "test-unit - run all tests",
+      "type": "Ruby",
+      "request": "launch",
+      "cwd": "${workspaceRoot}",
+      "program": "${workspaceRoot}/bin/rake",
+      "args": [
+        "test"
+      ],
+      "env": {
+        "TEST_SQL_URL": "postgres://localhost/couchdb_to_sql_test",
+        "COUCHDB_URL": "http://admin:admin@127.0.0.1:5984/"
+      }
+    },
+    {
+      "name": "test-unit - active spec file only",
+      "type": "Ruby",
+      "request": "launch",
+      "cwd": "${workspaceRoot}",
+      "program": "${workspaceRoot}/bin/rake",
+      "args": [
+        "test"
+      ],
+      "env": {
+        "TEST": "${file}",
+        "TEST_SQL_URL": "postgres://localhost/couchdb_to_sql_test",
+        "COUCHDB_URL": "http://admin:admin@127.0.0.1:5984/"
+      }
+    },
+    {
+      "name": "run with test_tap.rb",
+      "type": "Ruby",
+      "request": "launch",
+      "cwd": "${workspaceRoot}",
+      "program": "${workspaceRoot}/bin/bundler",
+      "args": [
+        "exec",
+        "${workspaceRoot}/exe/couchdb_to_sql",
+        "test_tap.rb"
+      ]
+    }
+  ]
+}

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec
+
+if defined?(JRUBY_VERSION)
+  gem 'jdbc-postgres'
+else
+  gem 'pg'
+  gem 'sqlite3'
+end

data/LICENSE

@@ -0,0 +1,24 @@
+The MIT License (MIT)
+
+Copyright © Sam Lown 2013-2016
+Copyright © eCraft Sverige AB 2017
+
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,163 @@
+[![Build Status](https://travis-ci.org/ecraft/couch_tap.svg?branch=master)](https://travis-ci.org/ecraft/couch_tap)
+
+# couchdb_to_sql
+
+Utility to listen to a CouchDB changes feed and automatically insert, update,
+or delete rows into an SQL database from matching key-value conditions of incoming documents.
+
+`couchdb_to_sql` is heavily indebted to [samlown's](https://github.com/samlown) original [couch_tap](https://github.com/samlown/couch_tap) gem. We have added functionality needed for our particular use case, while still trying to keep it reasonably flexible and not too hardwired to the `ember-pouch` use case.
+
+While CouchDB is awesome, business people probably won't be quite as impressed when they want to play around with the data. Regular SQL is generally accepted as being easy to use and much more widely supported by a larger range of commercial tools.
+
+`couchdb_to_sql` will listen to incoming documents on a CouchDB server's [_changes feed](http://docs.couchdb.org/en/2.1.0/api/database/changes.html) in continuous mode, and automatically update rows of the SQL database tables defined in the conversion schema. The changes feed uses a sequence number allowing synchronization to be started and stopped at will.
+
+[Sequel](http://sequel.jeremyevans.net/) is used to provide the connection to the database. This library can also be used for migrations, which is important for frequently changing schemas.
+
+`couchdb_to_sql` takes a simple two-step approach converting documents to rows. When a change event is received for a matching `document` definition, each associated row is completely deleted. If the change is anything other than a delete event, the rows will be re-created with the new data. This makes things much easier when trying to deal with multi-level documents (i.e. documents of documents) and one-to-many table relationships.
+
+
+## A `couchdb_to_sql` Project
+
+`couchdb_to_sql` requires a configuration or filter definition that will allow incoming document changes to be identified and dealt with. The configuration file can either be hand-written or generated dynamically. (For our particular use case with `ember-pouch`, we have chosen to generate it based on the Ember model metadata. The script for this is unfortunately not open source at this time.)
+
+The following example attempts to outline most of the key features of the DSL.
+
+```ruby
+# The couchdb database from which to request the changes feed
+changes "http://user:pass@host:port/invoicing" do
+  # # Optional flag which can be enabled to take advantage of Postgres 9.5's support for INSERT CONFLICT, e.g. upserts.
+  # # Note: this only deals with the _couchdb_to_sql_sequences metadata table, not the actual CouchDB documents themselves.
+  # upsert_mode
+
+  # # Optional flag which can be enabled if ember-pouch is being used to populate the CouchDB database. ember-pouch uses a
+  # # specially crafted format of the CouchDB documents, where all the data is placed in 'data' node and the 'id' follows a
+  # # particular format. This flag makes couchdb_to_sql presume that all CouchDB documents for the given stream follow this format.
+  # ember_pouch_mode
+
+  # # Optional flag which can be enabled to enable a stricter mode, where processing will abort if an unhandled document is
+  # # encountered.
+  # fail_on_unhandled_document
+
+  # # Optional path to a file containing a JSON array of sequences to skip. The 'seq' value of incoming documents will be compared
+  # # to the values in this array.
+  # skip_seqs_file 'skiplist.json'
+
+  # The target database to which changes will be streamed.
+  database "postgres://user:pass@localhost:5432/invoicing"
+
+  # Simple automated copy, each property's value in the matching CouchDB document will be copied to the table field with the
+  # same name.
+  document 'type' => 'User' do
+    table :users
+  end
+
+  document 'type' => 'Invoice' do
+
+    table :invoices, :key => :invoice_id do
+
+      # Copy columns from fields with different name
+      column :updated_at, :updated_on
+      column :created_at, :created_on
+
+      # Manually set a value from document or fixed variable
+      column :date, doc['date'].to_json
+      column :added_at, Time.now
+
+      # Set column values from a block.
+      column :total do
+        doc['items'].inject(0){ |sum,item| sum + item['total'] }
+      end
+    end
+  end
+end
+```
+
+## DSL Summary
+
+### changes
+
+Defines which CouchDB database should be used to request the changes feed.
+
+After loading the rest of the configuration, the service will connect to the database using Event Machine. As new changes come into the system, they will be managed in the background.
+
+
+### connection
+
+The Sequel URL used to connect to the destination database. Behind the scenes, `couchdb_to_sql` will check for a table named `couchdb_sequence` that contains a single row for the current changes sequence id, much like a migration id typically seen in a Rails database.
+
+As changes are received from CouchDB, the current sequence will be updated to match.
+
+#### document
+
+When a document is received from the changes feed, it will be passed through each
+`document` stanza looking for a match. Take the following example:
+
+    document :type => 'Invoice' do |doc|
+      # ...
+    end
+
+This will match all documents whose `type` property is equal to "Invoice". The document itself will be made available as a hash through the `doc` block variable.
+
+`document` stanzas may be nested if required to provide further levels of filtering.
+
+#### table
+
+Each `table` stanza lets `couchdb_to_sql` know that all or part of the current document should be inserted into it. By default, the matching table's schema will be read and any field names that match a property in the top-level of the document will be inserted automatically.
+
+One of the limitations of `couchdb_to_sql` is that all tables must have an id field as their primary key. In each row, the id's value will be copied from the `_id` of the document being imported. This is the only way that deleted documents can be reliably found and removed from the relational database.
+
+#### column
+
+#### foreign_key
+
+
+### Notes on deleted documents
+
+CouchDB documents being deleted are not deleted in the SQL database, because this is typically not what you want to do from a data integrity/etc. point of view. Instead, it is marked as deleted.
+
+For this to work, the following two columns must exist in the table (example given is from PostgreSQL):
+
+```
+fieldops_reports=# \d spare_parts
+                       Table "public.spare_parts"
+       Column       |           Type           |       Modifiers
+--------------------+--------------------------+------------------------
+ spare_part_id      | text                     | not null
+ id                 | text                     |
+ _deleted           | boolean                  | not null default false
+ _deleted_timestamp | timestamp with time zone |
+```
+
+(`spare_part_id` is the primary key which will hold the CouchDB id. `id` holds the "Ember ID" in case you are using `ember-pouch` mode. `_deleted*` are the fields which indicate if the record is deleted, and if so, when it was marked as deleted.)
+
+## Testing
+
+Run tests using rake, or individual tests as follows:
+
+```shell
+$ rake test TEST=test/unit/changes_test.rb
+```
+
+If you have disabled the "admin party" in CouchDB, you might have to manually specify the CouchDB URL. Like this:
+
+```shell
+$ COUCHDB_URL='http://admin:admin@127.0.0.1:5984/' bundle exec rake test
+```
+
+If you want to run tests towards a PostgreSQL database instead of CouchDB:
+
+```shell
+$ TEST_SQL_URL='postgres://localhost/couchdb_to_sql_test' bundle exec rake test
+```
+
+## Useful environment variables
+
+- `SEQUEL_LOG_LEVEL=debug` - set to enable logging of all SQL queries executed.
+
+## Releasing a new version
+
+- Merge all relevant pull requests
+- Bump the version in the `VERSION` file. Follow Semantic Versioning principles. Do not prepend the version with a v. You don't need to commit or push after this step, it gets done automatically by the next step.
+- `git release v1.0.x` (`brew install git-extras` if you are missing the `git release` command.)
+- `bundle exec rake build release` (builds the `.gem` file and pushes it to Rubygems.org)
+- `changelog-rs --latest` to regenerate the changelog which can then be copy-pasted to the [releases page](https://github.com/ecraft/couchdb_to_sql/releases). `curl https://sh.rustup.rs -sSf | sh && cargo install changelog-rs` if you don't have it installed. More info on [its web page](https://github.com/perlun/changelog-rs).

data/Rakefile

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'bundler'
+require 'rubygems'
+require 'rake/testtask'
+require 'rubocop/rake_task'
+
+Bundler::GemHelper.install_tasks
+RuboCop::RakeTask.new
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList.new('test/unit/**/*.rb')
+
+  # Without this setting, the unit test running generates a load of warnings in unrelated/3rd party gems, which obscures the
+  # real output of the test runs and makes it harder to read.
+  t.warning = false
+end
+
+# The tests are unfortunately at the moment MRI only, because of an Sqlite dependency:
+# https://github.com/ecraft/couchdb_to_sql/issues/9
+if defined?(JRUBY_VERSION)
+  desc 'Runs Rubocop linting'
+  task default: :rubocop
+else
+  desc 'Run Rubocop linting and the unit tests'
+  task default: %i[rubocop test]
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/couchdb_to_sql.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+  s.name          = 'couchdb_to_sql'
+  s.version       = `cat VERSION`.strip
+  s.date          = File.mtime('VERSION')
+  s.summary       = 'Listen to a CouchDB changes feed and create rows in a relational database in real-time.'
+  s.description   = "couchdb_to_sql provides a DSL that allows complex CouchDB documents to be converted into rows in a RDBMS' " \
+                    'table. The stream of events received from the CouchDB changes feed will trigger documents to be fed into a ' \
+                    'matching filter block and saved in the database.'
+  s.authors       = ['Sam Lown', 'Per Lundberg', 'Jens Nockert', 'Andreas Finne']
+  s.homepage      = 'https://github.com/ecraft/couchdb_to_sql'
+  s.license       = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.bindir        = 'exe'
+  s.executables   = `git ls-files -- exe/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.add_dependency 'activesupport', '~> 5.0'
+  s.add_dependency 'couchrest', '~> 2.0'
+  s.add_dependency 'httpclient', '~> 2.6'
+  s.add_dependency 'logging_library', '~> 1.0', '>= 1.0.5'
+  s.add_dependency 'sequel', '>= 4.36.0'
+
+  s.add_development_dependency 'mocha'
+  s.add_development_dependency 'rake', '~> 12.0'
+  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'simplecov', '~> 0.15'
+  s.add_development_dependency 'test-unit', '~> 3.2'
+end

data/examples/feed.rb

@@ -0,0 +1,22 @@
+
+# frozen_string_literal: true
+
+# Sample Configuration Script
+#
+# Run using the command line application:
+#
+#     couchdb_to_sql feed.rb
+#
+
+changes 'http://user:pass@host:port/invoicing' do
+  # Which database should we connect to?
+  database 'sqlite:///database.sqlite3'
+
+  filter 'type' => 'User' do
+    table :users
+  end
+
+  filter 'type' => 'Journey' do
+    table :journeys
+  end
+end