metacrunch-db 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2a1b93400bb9101dab9fe5000bcb94a64331aa5
-  data.tar.gz: c0f95abc70ce879f8a768bb91574eead74cb2f94
+  metadata.gz: d794b9e90321be6c7218516ae260742f3e9fdcbf
+  data.tar.gz: 65b4bbfb94017879c840b95ca8272a0c93beccac
 SHA512:
-  metadata.gz: b684069ae65e5adf3d0e4eaf11322184ffb26de4721d973f87e5030d629a0ab5568d6a5d9131ff245c8d8918486a95aeafd918bc3e945edc1da4aa32ba23e517
-  data.tar.gz: 303552d24d7f0b132a3e5cdaf4ae53b1aff6d051c0b860c534d1b3be44a3f437dc688faf78e7c8c16d45567fb0a46860ad6e8804314c857f14e74d2c077ed328
+  metadata.gz: 3e9682fa4db51b938c16474686b8110849ab7f05483cb70eaf0b753660fb7ea3cc493b6e674f4724e14213cfca7bb03c6d8975baf676f3474ec2c114be7a0245
+  data.tar.gz: 719564bf0c08fe07019316ac71618c94f5c6a1e829763899f92b622a4089cb81d5de69c193f18ee6b52d746dc1eb69e8f2e0276282dd1606b296f912032d071e

data/Readme.md

@@ -0,0 +1,86 @@
+metacrunch-db
+=============
+
+[![Gem Version](https://badge.fury.io/rb/metacrunch-db.svg)](http://badge.fury.io/rb/metacrunch-db)
+[![Code Climate](https://codeclimate.com/github/ubpb/metacrunch-db/badges/gpa.svg)](https://codeclimate.com/github/ubpb/metacrunch-db)
+[![Build Status](https://travis-ci.org/ubpb/metacrunch-db.svg)](https://travis-ci.org/ubpb/metacrunch-db)
+
+This is the official SQL database package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch). The implementation uses the [Sequel](https://github.com/jeremyevans/sequel) Gem as a dependency. Every database that is supported by Sequel can be used with this package.
+
+Installation
+------------
+
+Include the gem in your `Gemfile`
+
+```ruby
+gem "metacrunch-db", "~> 1.0.0"
+```
+
+and run `$ bundle install` to install it.
+
+Or install it manually
+
+```
+$ gem install metacrunch-db
+```
+
+
+Usage
+-----
+
+*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
+
+### Source
+
+The `Metacrunch::DB::Source` class can be used to read data from SQL databases into a metacrunch job.
+
+```ruby
+# my_job.metacrunch
+
+# Create a Sequel database connection 
+SOURCE_DB = Sequel.connect(...)
+
+# Create a Sequel dataset with an unambiguous order.
+my_source_dataset = SOURCE_DB[:my_table].order(:id)
+
+# Set the source
+source Metacrunch::DB::Source.new(my_source_dataset [, OPTIONS])
+```
+
+The implementation uses Sequel's [`paged_each`](http://www.rubydoc.info/github/jeremyevans/sequel/Sequel%2FDataset%3Apaged_each) to efficiently iterate even over large result sets. You can provide the following options, to control how `paged_each` works.
+
+**Options**
+
+For a detailed descriptions consult the Sequel documentation of [`paged_each`](http://www.rubydoc.info/github/jeremyevans/sequel/Sequel%2FDataset%3Apaged_each). Please note that the default for `strategy` has been changed to `:filter`.
+
+* `:rows_per_fetch`: Defaults to 1000.
+* `:strategy`: `:offset` or `:filter`, Defaults to `:filter`.
+* `:filter_values`: Defaults to `nil`
+
+
+### Destination
+
+The `Metacrunch::DB::Destination` class can be used to write data from a metacrunch job to SQL databases.
+
+```ruby
+# my_job.metacrunch
+
+# Create a Sequel database connection 
+DEST_DB = Sequel.connect(...)
+
+# Create a Sequel dataset where data should be written
+my_target_dataset = DEST_DB[:my_table]
+
+# For performance reasons it may be useful to create a batch
+# of records that gets written to the database
+transformation ->(row) { row }, batch_size: 1000
+
+# Set the destination
+destination Metacrunch::DB::Destination.new(my_target_dataset [, OPTIONS])
+```
+
+**Options**
+
+* `use_upsert`: When set to `true` it will perform an upsert (Update an existing record) and not an insert. Defaults to `false`.
+* `primary_key`: The primary key to use to identify an existing record in case of an upsert. It defaults to `:id`.
+* `transaction_options`: A hash of options to control how the database should handle the transaction. For a complete list of available options checkout out the Sequel documentation [here](http://www.rubydoc.info/github/jeremyevans/sequel/Sequel/Database#transaction-instance_method).

data/lib/metacrunch/db/destination.rb

@@ -6,8 +6,7 @@ module Metacrunch
     DEFAULT_OPTIONS = {
       use_upsert: false,
       primary_key: :id,
-      isolation: :repeatable,
-      num_retries: 5
+      transaction_options: {}
     }
 
     def initialize(sequel_dataset, options = {})
@@ -16,10 +15,9 @@ module Metacrunch
     end
 
     def write(data)
-      @dataset.db.transaction(
-        isolation: @options[:isolation],
-        num_retries: @options[:num_retries]
-      ) do
+      return if data.blank?
+
+      @dataset.db.transaction(@options[:transaction_options]) do
         if data.is_a?(Array)
           data.each{|d| insert_or_upsert(d) }
         else
@@ -35,11 +33,11 @@ module Metacrunch
   private
 
     def insert_or_upsert(data)
-      @use_upsert ? upsert(data) : insert(data)
+      @options[:use_upsert] ? upsert(data) : insert(data)
     end
 
     def insert(data)
-      @dataset.insert(data) if data
+      @dataset.insert(data)
     end
 
     def upsert(data)

data/lib/metacrunch/db/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module DB
-    VERSION = "1.0.0"
+    VERSION = "1.0.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-db
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - René Sprotte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-21 00:00:00.000000000 Z
+date: 2017-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -49,6 +49,7 @@ files:
 - Gemfile
 - License.txt
 - Rakefile
+- Readme.md
 - bin/console
 - lib/metacrunch/db.rb
 - lib/metacrunch/db/destination.rb