metacrunch 4.0.3 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6cd1ffde611aeb9af20078f87408261f3dfa1f85
-  data.tar.gz: 279a857dbb4907237a86deeac815feeebe9b20dc
+  metadata.gz: a192155704d21d4792eab70bb1122bcd92a14d84
+  data.tar.gz: 07cf95291da074427756f363621947baaa248d95
 SHA512:
-  metadata.gz: a3a0257a6bf2fc1cdac8ce1414f7b1c1185c218983412dce4ba0f98f2f8a4cd9719acc105c12e96e12ca32ebcbbaa1ac403774354b451c9044697833d087596f
-  data.tar.gz: f16c860f681612118e43213406e5b034cf648a21574d568d655c9e98d8ac2b29a70eb66333dbedde0bfb9a55a1566dbdbaee2000731e9eefd377a50586047369
+  metadata.gz: 1f360eebf44dc1b2a3025194c592e4bf01787c6637feec58aba758826c8d73eb4ec119730f539321911237cd395172b1c4a16cbdcc5d2728ed3830cc5fe64957
+  data.tar.gz: fa06ae98c1a2d4d8d7b4c7eaad845526df3ee5c12bbd1c6902a5ac90e9c32c1bab43b4886d13a40b691f7bff9c09d745ef60e61ac038a8163534b6fd307c6471

data/Readme.md

@@ -102,19 +102,27 @@ transformation MyTransformation.new
 
 Sometimes it is useful to buffer data between transformation steps to allow a transformation to work on larger bulks of data. metacrunch uses a simple transformation buffer to achieve this.
 
-To use a transformation buffer pass the buffer size as an option to the transformation.
+To use a transformation buffer add the `:buffer` option to your transformation. You can pass a positive integer value as a buffer size, or as an advanced option you can pass a `Proc` object. The buffer flushes every time the buffer reaches the given size or if the `Proc` returns `true`.
 
 ```ruby
 # File: my_etl_job.metacrunch
 
 source 1..95 # A range responds to #each and is a valid source
 
+# A buffer with a fixed size
 transformation ->(bulk) { 
   # this transformation is called when the buffer 
   # is filled with 10 objects or if the source has
   # yielded the last data object.
   # bulk would be: [1,...,10], [11,...,20], ..., [91,...,95]
-}, buffer_size: 10
+}, buffer: 10
+
+# A buffer that uses a Proc
+transformation ->(bulk) { 
+  # ...
+}, buffer: -> {
+  true if some_condition
+}
 ```
 
 #### Defining a destination
@@ -163,7 +171,7 @@ In this example we declare two options `log_level` and `database_url`. `log_leve
 To set/override these options use the command line.
 
 ```
-$ bundle exec metacrunch my_etl_job.metacrunch --log-level debug
+$ metacrunch my_etl_job.metacrunch --log-level debug
 ```
 
 This will set the `options[:log_level]` to `debug`.
@@ -171,9 +179,8 @@ This will set the `options[:log_level]` to `debug`.
 To get a list of available options for a job, use `--help` on the command line.
 
 ```
-$ bundle exec metacrunch my_etl_job.metacrunch --help
+$ metacrunch my_etl_job.metacrunch --help
 
-Usage: metacrunch [options] JOB_FILE [job-options] [ARGS]
 Job options:
     -l, --log-level LEVEL            Log level (debug,info,warn,error)
                                      DEFAULT: info
@@ -215,12 +222,17 @@ If you use [Bundler](http://bundler.io) to manage dependencies for your jobs mak
 $ bundle exec metacrunch my_etl_job.metacrunch
 ```
 
-Depending on your environment `bundle exec` may not be required (e.g. if you have rubygems-bundler installed) but we recommend using it whenever you have a Gemfile you like to use. When using Bundler make sure to add `gem "metacrunch"` to the Gemfile.
+In your job file use `Bundler.require` to require the dependencies from your `Gemfile`.
+
+```ruby
+# File: my_etl_job.metacrunch
+Bundler.require
+```
 
 Use the following syntax to run a metacrunch job
 
 ```
-$ [bundle exec] metacrunch [COMMAND_OPTIONS] JOB_FILE [JOB_OPTIONS] [JOB_ARGS...]
+$ [bundle exec] metacrunch [options] JOB_FILE [job-options] [ARGS...]
 ```
 
 
@@ -345,6 +357,15 @@ destination MyDestination.new
 
 ```
 
+Official extension packages
+---------------------------
+
+* [metacrunch-db](https://github.com/ubpb/metacrunch-db): SQL Database package
+* [metacrunch-file](https://github.com/ubpb/metacrunch-file): File package
+* [metacrunch-elasticsearch](https://github.com/ubpb/metacrunch-elasticsearch): [Elasticsearch](https://www.elastic.co) package
+* [metacrunch-redis](https://github.com/ubpb/metacrunch-redis): [Redis](https://redis.io) package
+* [metacrunch-marcxml](https://github.com/ubpb/metacrunch-marcxml): [MARCXML](http://www.loc.gov/standards/marcxml/) package
+
 Upgrading
 ---------
 
@@ -353,7 +374,7 @@ Upgrading
 When upgrading from metacrunch 3.x, there are some breaking changes you need to address.
 
 * There is now only one `source` and `destination`. If you have more than one in your job file the last definition will used.
-* There is no `transformation_buffer` anymore. Instead set `buffer_size` as an option to `transformation`.
+* There is no `transformation_buffer` anymore. Instead set `buffer` as an option to `transformation`.
 * `transformation`, `pre_process` and `post_process` can't be implemented using a block anymore. Always use a `callable` (E.g. Lambda, Proc or any object responding to `#call`).
 * When running jobs via the CLI you do not need to separate the arguments passed to metacrunch from the arguments passed to the job with `@@`.
 * The `args` function used to get the non-option arguments passed to a job has been removed. Use `ARGV` instead.

data/lib/metacrunch/job.rb

@@ -14,6 +14,8 @@ module Metacrunch
     def initialize(file_content = nil, &block)
       @dsl = Dsl.new(self)
 
+      @deprecator = ActiveSupport::Deprecation.new("5.0.0", "metacrunch")
+
       if file_content
         @dsl.instance_eval(file_content, "Check your metacrunch Job at Line")
       elsif block_given?
@@ -61,11 +63,16 @@ module Metacrunch
       @transformations ||= []
     end
 
-    def add_transformation(callable, buffer_size: nil)
+    def add_transformation(callable, buffer_size: nil, buffer: nil)
       ensure_callable!(callable)
 
-      if buffer_size && buffer_size.to_i > 0
-        transformations << Metacrunch::Job::Buffer.new(buffer_size)
+      if buffer_size && buffer_size.is_a?(Numeric)
+        @deprecator.deprecation_warning(:buffer_size, :buffer)
+        buffer = buffer_size
+      end
+
+      if buffer
+        transformations << Metacrunch::Job::Buffer.new(buffer)
       end
 
       transformations << callable
@@ -120,11 +127,13 @@ module Metacrunch
     def run_transformations(data, flush_buffers: false)
       transformations.each do |transformation|
         if transformation.is_a?(Buffer)
+          buffer = transformation
+
           if data
-            data = transformation.buffer(data)
-            data = transformation.flush if flush_buffers
+            data = buffer.buffer(data)
+            data = buffer.flush if flush_buffers
           else
-            data = transformation.flush
+            data = buffer.flush
           end
         else
           data = transformation.call(data) if data

data/lib/metacrunch/job/buffer.rb

@@ -1,25 +1,30 @@
 module Metacrunch
   class Job::Buffer
 
-    def initialize(size)
-      @size = size
+    def initialize(size_or_proc)
+      @size_or_proc = size_or_proc
+      @buffer = []
+
+      if @size_or_proc.is_a?(Numeric) && @size_or_proc <= 0
+        raise ArgumentError, "Buffer size must be a posive number greater that 0."
+      end
     end
 
     def buffer(data)
-      storage << data
-      flush if storage.count >= @size
+      @buffer << data
+
+      case @size_or_proc
+      when Numeric
+        flush if @buffer.count >= @size_or_proc.to_i
+      when Proc
+        flush if @size_or_proc.call == true
+      end
     end
 
     def flush
-      storage
+      @buffer
     ensure
-      @buffer = nil
-    end
-
-  private
-
-    def storage
-      @buffer ||= []
+      @buffer = []
     end
 
   end

data/lib/metacrunch/job/dsl.rb

@@ -22,8 +22,8 @@ module Metacrunch
       @_job.post_process = callable
     end
 
-    def transformation(callable, buffer_size: nil)
-      @_job.add_transformation(callable, buffer_size: buffer_size)
+    def transformation(callable, buffer_size: nil, buffer: nil)
+      @_job.add_transformation(callable, buffer_size: buffer_size, buffer: buffer)
     end
 
     def options(require_args: false, &block)

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "4.0.3"
+  VERSION = "4.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.1.0
 platform: ruby
 authors:
 - René Sprotte
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-26 00:00:00.000000000 Z
+date: 2017-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport