metacrunch 2.2.3 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 071c720dfc2ad17aa71665f25579961eb1a0beea
-  data.tar.gz: 622e3192f6021da9e5851485b7fb9181ec75e2c6
+  metadata.gz: 657c0cedb16066bee69a76a30a0e02783e5d0209
+  data.tar.gz: 7c0290c94a11af5daf8bfb40e0b0abab3f92c8df
 SHA512:
-  metadata.gz: 6fe42cf29e183659f667cfac082cebebee345ab199a4fd99dd175a0ea1716ea3a5b38b3a4ce9ab13c790a4dc03d4775d343d55fe0593b43ba10eaac5557654c9
-  data.tar.gz: f14bee54085db41b4ed15a6f551a9f6a1465541c896037a3674cafff4cbbab375574c0236b2e9c33b9c03e3fbef929d11e0aa473b11f4a8f763ef001c371e879
+  metadata.gz: ee0834cbe3f70238f1c5f7fbaf37284c0c7adedbdafe4c01e7b31819cfa8274585a4cd8407fd5cc35ca32c033f0944c86f872d09d8ee5d56ba22d48435898d4d
+  data.tar.gz: d9663a2b36247dcd20721e9e2016bc3f1d42dae47aa89acc671fb112437c925779bf5949edfb5409e3e6ff3aa2612216c93468847deb51ebc8b4ae6da064eb97

data/.travis.yml

@@ -1,3 +1,4 @@
 language: ruby
 rvm:
-  - "2.2.2"
+  - ruby-2.3.0
+  - jruby-9.0.5.0

data/Gemfile

@@ -1,25 +1,23 @@
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in your gemspec
 gemspec
 
 group :development do
-  gem "bundler"
-  gem "nokogiri"
-  gem "rake"
-  gem "rspec",           ">= 3.0.0",  "< 4.0.0"
-  gem "simplecov",       ">= 0.8.0"
+  gem "bundler",      ">= 1.7"
+  gem "rake",         ">= 11.1"
+  gem "rspec",        ">= 3.0.0",  "< 4.0.0"
+  gem "simplecov",    ">= 0.11.0"
+  gem "sqlite3",      ">= 1.3.11", platform: :ruby
+  gem "jdbc-sqlite3", ">= 3.8", platform: :jruby
 
   if !ENV["CI"]
-    gem "hashdiff"
-    gem "pry",                "~> 0.9.12.6"
-    gem "pry-byebug",         "<= 1.3.2"
-    gem "pry-rescue",         "~> 1.4.2"
-    gem "pry-stack_explorer", "~> 0.4.9.1"
-    gem "pry-syntax-hacks",   "~> 0.0.6"
+    gem "hashdiff",   ">= 0.3.0", platform: :ruby
+    gem "pry-byebug", ">= 3.3.0", platform: :ruby
+    gem "pry-rescue", ">= 1.4.2", platform: :ruby
+    gem "pry-state",  ">= 0.1.7", platform: :ruby
   end
 end
 
 group :test do
-  gem "codeclimate-test-reporter", require: nil
+  gem "codeclimate-test-reporter", ">= 0.5.0", require: nil
 end

data/License.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 René Sprotte, Michael Sievers, Marcel Otto
+Copyright (c) 2016 René Sprotte, Michael Sievers, Marcel Otto
 
 MIT License
 

data/Readme.md

@@ -1,7 +1,144 @@
-# metacrunch
+metacrunch
+==========
 
 [![Gem Version](https://badge.fury.io/rb/metacrunch.svg)](http://badge.fury.io/rb/metacrunch)
 [![Code Climate](https://codeclimate.com/github/ubpb/metacrunch/badges/gpa.svg)](https://codeclimate.com/github/ubpb/metacrunch)
 [![Build Status](https://travis-ci.org/ubpb/metacrunch.svg)](https://travis-ci.org/ubpb/metacrunch)
 
-Dokumentation folgt in Kürze...
+metacrunch is a simple and lightweight data processing and ETL ([Extract-Transform-Load](http://en.wikipedia.org/wiki/Extract,_transform,_load))
+toolkit for Ruby.
+
+
+Installation
+------------
+
+```
+$ gem install metacrunch
+```
+
+
+Create ETL jobs
+---------------
+
+The basic idea behind an ETL job in metacrunch is the concept of a data processing pipeline. Each ETL job reads data from one or more **sources** (extract step), runs one or more **transformations** (transform step) on the data and finally writes the transformed data back to one or more **destinations** (load step).
+
+metacrunch provides you with a simple DSL to define such ETL jobs. Just create a text file with the extension `.metacrunch`. Note: The extension doesn't really matter but you should avoid `.rb` to not loading them by mistake from another Ruby component.
+
+Let's take a look at an example. For a collection of working examples check out our [metacrunch-demo](https://github.com/ubpb/metacrunch-demo) repo.
+
+```ruby
+# File: my_etl_job.metacrunch
+
+# Every metacrunch job file is a regular Ruby file. So you can always use regular Ruby
+# stuff like declaring methods
+def my_helper
+  # ...
+end
+
+# ... declaring classes
+class MyHelper
+  # ...
+end
+
+# ... declaring variables
+foo = "bar"
+
+# ... or loading other ruby files
+require_relative "./some/other/ruby/file"
+
+# Declare a source (use a build-in or 3rd party source or implement it – see notes below).
+# At least one source is required to allow the job to run.
+source MySource.new
+# ... maybe another one. Sources are processed in the order they are defined.
+source MyOtherSource.new
+
+# Declare a destination (use a build-in or 3rd party destination or implement it – see notes below).
+# Technically a destination is optional, but a job that doesn't store it's
+# output doesn't really makes sense.
+destination MyDestination.new
+# ... you can have more destinations if you like
+destination MyOtherDestination.new
+
+# To process data use the #transformation hook.
+transformation do |data|
+  # Called for each data object that has been put in the pipeline by a source.
+
+  # Do your data transformation process here.
+
+  # You must return the data to keep it in the pipeline. Dismiss the
+  # data conditionally by returning nil.
+  data
+end
+
+# Instead of passing a block to #transformation you can pass a
+# `callable` object (an object responding to #call).
+transformation Proc.new {
+  # Procs and Lambdas responds to #call
+}
+
+# MyTransformation defines #call
+transformation MyTransformation.new
+
+# To run arbitrary code before the first transformation use the #pre_process hook.
+pre_process do
+  # Called before the first transformation
+end
+
+# To run arbitrary code after the last transformation use the #post_process hook.
+post_process do
+  # Called after the last transformation
+end
+
+# Instead of passing a block to #pre_process or #post_process you can pass a
+# `callable` object (an object responding to #call).
+pre_process Proc.new {
+  # Procs and Lambdas responds to #call
+}
+
+# MyCallable class defines #call
+post_process MyCallable.new
+
+```
+
+
+Run ETL jobs
+------------
+
+metacrunch comes with a handy command line tool. In your terminal just call
+
+
+```
+$ metacrunch run my_etl_job.metacrunch
+```
+
+to run the job.
+
+Implementing sources
+--------------------
+
+TBD.
+
+Implementing transformations
+----------------------------
+
+TBD.
+
+Implementing writers
+---------------------
+
+TBD.
+
+Defining job dependencies
+-------------------------
+
+TBD.
+
+Defining job options
+--------------------
+
+TBD.
+
+License
+-------
+
+metacrunch is available at [github](https://github.com/ubpb/metacrunch) under [MIT license](https://github.com/ubpb/metacrunch/blob/master/License.txt).

data/bin/console

@@ -1,11 +1,14 @@
 #!/usr/bin/env ruby
-
 require "bundler/setup"
 require "metacrunch"
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
+begin
+  require "pry"
+rescue LoadError ; end
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-require "pry"
-Pry.start
+if defined?(Pry)
+  Pry.start
+else
+  require "irb"
+  IRB.start
+end

data/exe/metacrunch

@@ -1,4 +1,3 @@
 #!/usr/bin/env ruby
 require "metacrunch"
-Metacrunch.load_plugins
-Metacrunch::Cli.start(ARGV)
+Metacrunch::Cli.new.run

data/lib/metacrunch/cli.rb

@@ -1,26 +1,74 @@
 module Metacrunch
   class Cli
-    require_relative "./cli/main"
-    require_relative "./cli/base"
-    require_relative "./cli/command_registry"
-    require_relative "./cli/command_definition"
+    ARGS_SEPERATOR = "@@"
 
-    def self.start(argv)
-      Main.start(argv)
+    def run
+      init_commander!
+      init_run_command!
+      run_commander!
     end
 
-    def self.setup(namespace, description, &block)
-      klass = Class.new(Base)
-      klass.namespace(namespace)
+  private
+    def commander
+      @commander ||= Commander::Runner.new(metacrunch_args)
+    end
+
+    def init_commander!
+      commander.program :name, "metacrunch"
+      commander.program :version, Metacrunch::VERSION
+      commander.program :description, "Data processing and ETL toolkit for Ruby."
+      commander.default_command :help
+    end
+
+    def run_commander!
+      commander.run!
+    end
+
+    def init_run_command!
+      commander.command :run do |c|
+        c.syntax = "metacrunch run [options] FILE [@@ job_options]"
+        c.description = "Runs a metacrunch job description."
+
+        c.action do |filenames, program_options|
+          if filenames.empty?
+            say "You need to provide a job description file."
+            exit(1)
+          elsif filenames.count > 1
+            say "You must provide exactly one job description file."
+          else
+            filename = File.expand_path(filenames.first)
+            dir = File.dirname(filename)
 
-      registry = CommandRegistry.new
-      yield(registry)
+            setup_bundler(dir)
 
-      registry.commands.each do |c|
-        klass.register_thor_command(c)
+            Dir.chdir(dir) do
+              contents = File.read(filename)
+              context = Metacrunch::Job.define(contents, filename: filename, args: job_args)
+              context.run
+            end
+          end
+        end
       end
+    end
+
+    def metacrunch_args
+      index = ARGV.index(ARGS_SEPERATOR)
+      @metacrunch_args ||= index ? ARGV[0..index-1] : ARGV
+    end
 
-      Main.register(klass, namespace, "#{namespace} [COMMAND]", description)
+    def job_args
+      index = ARGV.index(ARGS_SEPERATOR)
+      @job_args ||= index ? ARGV[index+1..-1] : nil
     end
+
+    def setup_bundler(dir)
+      ENV['BUNDLE_GEMFILE'] ||= File.join(dir, "Gemfile")
+      if File.exists?(ENV['BUNDLE_GEMFILE'])
+        puts "Using Gemfile `#{ENV['BUNDLE_GEMFILE']}`."
+        Bundler.setup
+        Bundler.require
+      end
+    end
+
   end
 end

data/lib/metacrunch/db/reader.rb

@@ -0,0 +1,27 @@
+module Metacrunch
+  class Db::Reader
+
+    def initialize(database_connection_or_url, dataset_proc, options = {})
+      @rows_per_fetch = options.delete(:rows_per_fetch) || 1000
+
+      @db = if database_connection_or_url.is_a?(String)
+        Sequel.connect(database_connection_or_url, options)
+      else
+        database_connection_or_url
+      end
+
+      @dataset = dataset_proc.call(@db)
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      @dataset.paged_each(rows_per_fetch: @rows_per_fetch) do |row|
+        yield(row)
+      end
+
+      self
+    end
+
+  end
+end

data/lib/metacrunch/db/writer.rb

@@ -0,0 +1,23 @@
+module Metacrunch
+  class Db::Writer
+
+    def initialize(database_connection_or_url, dataset_proc, options = {})
+      @db = if database_connection_or_url.is_a?(String)
+        Sequel.connect(database_connection_or_url, options)
+      else
+        database_connection_or_url
+      end
+
+      @dataset = dataset_proc.call(@db)
+    end
+
+    def write(data)
+      @dataset.insert(data)
+    end
+
+    def close
+      @db.disconnect
+    end
+
+  end
+end

data/lib/metacrunch/db.rb

@@ -0,0 +1,8 @@
+require "sequel"
+
+module Metacrunch
+  class Db
+    require_relative "db/reader"
+    require_relative "db/writer"
+  end
+end

data/lib/metacrunch/fs/entry.rb

@@ -0,0 +1,17 @@
+module Metacrunch
+  class Fs::Entry
+
+    attr_reader :filename, :archive_filename, :contents
+
+    def initialize(filename:, archive_filename: nil, contents: nil)
+      @filename         = filename
+      @archive_filename = archive_filename.presence
+      @contents         = contents
+    end
+
+    def from_archive?
+      @archive_filename != nil
+    end
+
+  end
+end

data/lib/metacrunch/{file_reader.rb → fs/reader.rb}

@@ -1,10 +1,9 @@
 require "rubygems/package"
-require_relative "./file_reader_entry"
 
 module Metacrunch
-  class FileReader
+  class Fs::Reader
 
-    def initialize(filenames)
+    def initialize(filenames = nil)
       @filenames = [*filenames].map{|f| f.presence}.compact
     end
 
@@ -23,27 +22,27 @@ module Metacrunch
   private
 
     def is_archive?(filename)
-      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz")
+      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz") || filename.ends_with?(".tgz")
     end
 
     def is_gzip_file?(filename)
-      filename.ends_with?(".gz")
+      filename.ends_with?(".gz") || filename.ends_with?(".tgz")
     end
 
     def read_regular_file(filename, &block)
-      if ::File.file?(filename)
-        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
-        yield Entry.new(filename: filename, archive_filename: nil, contents: io.read)
+      if File.file?(filename)
+        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : File.open(filename, "r")
+        yield Fs::Entry.new(filename: filename, archive_filename: nil, contents: io.read)
       end
     end
 
     def read_archive(filename, &block)
-      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
+      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : File.open(filename, "r")
       tarReader = Gem::Package::TarReader.new(io)
 
       tarReader.each do |_tar_entry|
         if _tar_entry.file?
-          yield Entry.new(
+          yield Fs::Entry.new(
             filename: filename,
             archive_filename: _tar_entry.full_name,
             contents: _tar_entry.read

data/lib/metacrunch/fs.rb

@@ -0,0 +1,6 @@
+module Metacrunch
+  module Fs
+    require_relative "fs/reader"
+    require_relative "fs/entry"
+  end
+end

data/lib/metacrunch/job/buffer.rb

@@ -0,0 +1,26 @@
+module Metacrunch
+  class Job::Buffer
+
+    def initialize(size)
+      @size = size
+    end
+
+    def buffer(data)
+      storage << data
+      flush if storage.count >= @size
+    end
+
+    def flush
+      storage
+    ensure
+      @buffer = nil
+    end
+
+  private
+
+    def storage
+      @buffer ||= []
+    end
+
+  end
+end

data/lib/metacrunch/job/dsl/option_support.rb

@@ -0,0 +1,102 @@
+module Metacrunch
+  class Job::Dsl::OptionSupport
+
+    def register_options(args, require_args: false, &block)
+      options = {}
+      registry.instance_eval(&block)
+
+      registry.each do |key, opt_def|
+        # Set default value
+        options[key] = opt_def[:default]
+
+        # Register with OptionParser
+        if opt_def[:args].present?
+          option = parser.define(*opt_def[:args]) { |value| options[key] = value }
+
+          option.desc << "REQUIRED" if opt_def[:required]
+          option.desc << "DEFAULT: #{opt_def[:default]}" if opt_def[:default].present?
+
+          parser_options[key] = option
+        end
+      end
+
+      # Finally parse CLI options with OptionParser
+      args = parser.parse(args || [])
+
+      # Make sure required options are present
+      ensure_required_options!(options)
+
+      # Make sure args are present if required
+      ensure_required_args!(args) if require_args
+
+      options
+    end
+
+  private
+
+    def parser
+      @parser ||= OptionParser.new do |parser|
+        parser.banner = "Usage: metacrunch run [options] JOB_FILE @@ [job-options] [ARGS]\nJob options:"
+      end
+    end
+
+    def parser_options
+      @parser_options ||= {}
+    end
+
+    def registry
+      @registry ||= OptionRegistry.new
+    end
+
+    def ensure_required_options!(options)
+      registry.each do |key, opt_def|
+        if opt_def[:required] && options[key].blank?
+          long_option = parser_options[key].long.try(:[], 0)
+          short_option = parser_options[key].short.try(:[], 0)
+
+          puts "Error: Required job option `#{long_option || short_option}` missing."
+          puts parser.help
+
+          exit(1)
+        end
+      end
+    end
+
+    def ensure_required_args!(args)
+      if args.blank?
+        puts "Error: Required ARGS are missing."
+        puts parser.help
+
+        exit(1)
+      end
+    end
+
+  private
+
+    class OptionRegistry
+
+      def add(name, *args, default: nil, required: false)
+        if default && required
+          raise ArgumentError, "You can't use `default` and `required` option at the same time."
+        end
+
+        options[name.to_sym] = {
+          args: args,
+          default: default,
+          required: required
+        }
+      end
+
+      def each(&block)
+        options.each(&block)
+      end
+
+    private
+
+      def options
+        @options ||= {}
+      end
+    end
+
+  end
+end

data/lib/metacrunch/job/dsl.rb

@@ -0,0 +1,42 @@
+module Metacrunch
+  class Job::Dsl
+    require_relative "dsl/option_support"
+
+    def initialize(job)
+      @_job = job
+    end
+
+    def source(source)
+      @_job.add_source(source)
+    end
+
+    def destination(destination)
+      @_job.add_destination(destination)
+    end
+
+    def pre_process(callable = nil, &block)
+      @_job.add_pre_process(callable, &block)
+    end
+
+    def post_process(callable = nil, &block)
+      @_job.add_post_process(callable, &block)
+    end
+
+    def transformation_buffer(size)
+      @_job.add_transformation_buffer(size)
+    end
+
+    def transformation(callable = nil, &block)
+      @_job.add_transformation(callable, &block)
+    end
+
+    def options(require_args: false, &block)
+      if block_given?
+        @_options = OptionSupport.new.register_options(@_job.args, require_args: require_args, &block)
+      else
+        @_options ||= {}
+      end
+    end
+
+  end
+end