kraps 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15d08cf8952d4e5a083a6a4f9791fd16e9e2dbf67c1c71326f3af840c0c72eb8
-  data.tar.gz: c6542584846c54e5897b7b59ef40e8ec282ee27521b2d5ff39551eb02755882d
+  metadata.gz: 921ae08326c96216136418861b88af7f11bce519c924cd1813216165f7f02690
+  data.tar.gz: '0913d31d3caeea0be664bc714e9d0da58227f515c047be31359e96040bc0c141'
 SHA512:
-  metadata.gz: 21d1ef7a132edacf54e0b2df12b8d085af84ec1ed1cd019d258e43aba4cffbecdeada9b2b7f4baeefec4b59d115eb3e38400da94a3d7961ab19bbbb7dd2cf58c
-  data.tar.gz: fde066e9fdc5f9df7e95be43142cb04a7a1c5279decb277f1d815db508c87d2c04be46ea9559069c8a2c9539ee2eaa949a2fe2fdc3bf862937f9211cdfd8fbd5
+  metadata.gz: d8e43e5229fc310019801e62a2e278470a1eb37b50e4aca27b9c64edb6666115f0f25c7a7375790516e2726fcf10980cdac1523c54dde8d3527a39fd919a2a5a
+  data.tar.gz: 30b1a9edcdd4f7ff476bfa4c070aef31debd727500e27a08b59f1df2663362c60e3cc3a3c860455d568abd994bb56a216f7eedf8baea6cc06ca73b1d0bdf9a07

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## v0.5.0
+
+* Added a `before` option to specify a callable to run before
+  a step to e.g. populate caches upfront, etc.
+
 ## v0.4.0
 
 * Pre-reduce in a map step when the subsequent step is a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kraps (0.4.0)
+    kraps (0.5.0)
       attachie
       distributed_job
       map-reduce-ruby (>= 3.0.0)

data/README.md

@@ -265,6 +265,19 @@ job.each_partition do |partition, pairs|
 end
 ```
 
+Please note that every API method accepts a `before` callable:
+
+```ruby
+before_block = proc do
+  # runs once before the map action in every worker, which can be useful to
+  # e.g. populate caches etc.
+end
+
+job.map(before: before_block) do |key, value, collector|
+  # ...
+end
+```
+
 ## More Complex Jobs
 
 Please note that a job class can return multiple jobs and jobs can build up on

data/lib/kraps/job.rb

@@ -9,46 +9,74 @@ module Kraps
       @partitioner = HashPartitioner.new
     end
 
-    def parallelize(partitions:, partitioner: HashPartitioner.new, worker: @worker, &block)
+    def parallelize(partitions:, partitioner: HashPartitioner.new, worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do
           @partitions = partitions
           @partitioner = partitioner
 
-          @steps << Step.new(action: Actions::PARALLELIZE, args: { partitions: @partitions, partitioner: @partitioner, worker: worker }, block: block)
+          @steps << Step.new(
+            action: Actions::PARALLELIZE,
+            partitions: @partitions,
+            partitioner: @partitioner,
+            worker: worker,
+            before: before,
+            block: block
+          )
         end
       end
     end
 
-    def map(partitions: nil, partitioner: nil, worker: @worker, &block)
+    def map(partitions: nil, partitioner: nil, worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do
           @partitions = partitions if partitions
           @partitioner = partitioner if partitioner
 
-          @steps << Step.new(action: Actions::MAP, args: { partitions: @partitions, partitioner: @partitioner, worker: worker }, block: block)
+          @steps << Step.new(
+            action: Actions::MAP,
+            partitions: @partitions,
+            partitioner: @partitioner,
+            worker: worker,
+            before: before,
+            block: block
+          )
         end
       end
     end
 
-    def reduce(worker: @worker, &block)
+    def reduce(worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do
-          @steps << Step.new(action: Actions::REDUCE, args: { partitions: @partitions, partitioner: @partitioner, worker: worker }, block: block)
+          @steps << Step.new(
+            action: Actions::REDUCE,
+            partitions: @partitions,
+            partitioner: @partitioner,
+            worker: worker,
+            before: before,
+            block: block
+          )
         end
       end
     end
 
-    def each_partition(worker: @worker, &block)
+    def each_partition(worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do
-          @steps << Step.new(action: Actions::EACH_PARTITION, args: { partitions: @partitions, partitioner: @partitioner, worker: worker }, block: block)
+          @steps << Step.new(
+            action: Actions::EACH_PARTITION,
+            partitions: @partitions,
+            partitioner: @partitioner,
+            worker: worker,
+            before: before,
+            block: block
+          )
         end
       end
     end
 
-    def repartition(partitions:, partitioner: nil, worker: @worker)
-      map(partitions: partitions, partitioner: partitioner, worker: worker) do |key, value, collector|
+    def repartition(partitions:, partitioner: nil, worker: @worker, before: nil)
+      map(partitions: partitions, partitioner: partitioner, worker: worker, before: before) do |key, value, collector|
         collector.call(key, value)
       end
     end

data/lib/kraps/runner.rb

@@ -55,7 +55,7 @@ module Kraps
             enqueue(token: distributed_job.token, part: part, item: item)
           end
 
-          Frame.new(token: distributed_job.token, partitions: @step.args[:partitions])
+          Frame.new(token: distributed_job.token, partitions: @step.partitions)
         end
       end
 
@@ -65,7 +65,7 @@ module Kraps
             enqueue(token: distributed_job.token, part: part, partition: partition)
           end
 
-          Frame.new(token: distributed_job.token, partitions: @step.args[:partitions])
+          Frame.new(token: distributed_job.token, partitions: @step.partitions)
         end
       end
 
@@ -75,7 +75,7 @@ module Kraps
             enqueue(token: distributed_job.token, part: part, partition: partition)
           end
 
-          Frame.new(token: distributed_job.token, partitions: @step.args[:partitions])
+          Frame.new(token: distributed_job.token, partitions: @step.partitions)
         end
       end
 
@@ -91,7 +91,7 @@ module Kraps
 
       def enqueue(token:, part:, **rest)
         Kraps.enqueuer.call(
-          @step.args[:worker],
+          @step.worker,
           JSON.generate(
             job_index: @job_index,
             step_index: @step_index,

data/lib/kraps/step.rb

@@ -1,3 +1,3 @@
 module Kraps
-  Step = Struct.new(:action, :args, :block, :frame, keyword_init: true)
+  Step = Struct.new(:action, :partitioner, :partitions, :block, :worker, :before, :frame, keyword_init: true)
 end

data/lib/kraps/version.rb

@@ -1,3 +1,3 @@
 module Kraps
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/kraps/worker.rb

@@ -13,6 +13,8 @@ module Kraps
       raise(InvalidAction, "Invalid action #{step.action}") unless Actions::ALL.include?(step.action)
 
       with_retries(retries) do # TODO: allow to use queue based retries
+        step.before&.call
+
         send(:"perform_#{step.action}")
 
         distributed_job.done(@args["part"])
@@ -194,7 +196,7 @@ module Kraps
     end
 
     def partitioner
-      @partitioner ||= proc { |key| step.args[:partitioner].call(key, step.args[:partitions]) }
+      @partitioner ||= proc { |key| step.partitioner.call(key, step.partitions) }
     end
 
     def distributed_job

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kraps
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-11-09 00:00:00.000000000 Z
+date: 2022-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attachie