kraps 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b24f67ff2122dc82d372eb421c379ac42f415674999958f42f71a2cdbee1a33
-  data.tar.gz: d8297482eb38a30cb8ff6a7761a544833e3795a8f3a045343f6a10c370d886ca
+  metadata.gz: 6ac0098a3c2f8acf49552f5b72621619345f9b59c83edd58368d7a999605f817
+  data.tar.gz: 3d522dd1149b57c9dce596ea4b38aa67ba5f2bc18a96bbc2840e80096e9eb192
 SHA512:
-  metadata.gz: 5543d1a8af8fa12007d38f00d9aa515eb1edf254d8bbc2aa8c133c0101dbe7ccaef5f0330929b6b84456a81eec7e5744091deace2d16ed2826f14cd56432db8f
-  data.tar.gz: 14992d608157562da3af98207681a66cf0a9a9566861fe79c949ccdec9db6cb344bfb070147fc820eac70b076c15db373896f80b5bd8d5f8714f9ce75d7eb7c8
+  metadata.gz: e859a2ecd0e635a067bde3cf3d51f76dc64702d91975342c590a43370d5a8a49da5c2272b6806550420537c74b3c7129ee0c76f58ff42ef0b154d7e8d0904417
+  data.tar.gz: 8e93aa69360b48b46f96a94cc2dd49d841b72fc858688c93071d46b99b86d50bf3b0758dd7f071cb50fd6ddc13cf532ed00d1ed1e4eee277d064568a11931f22

data/.rubocop.yml

@@ -54,6 +54,9 @@ Style/StringLiteralsInInterpolation:
   Enabled: true
   EnforcedStyle: double_quotes
 
+Style/RedundantSelfAssignment:
+  Enabled: false
+
 Layout/LineLength:
   Max: 250
 

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## v0.10.0
+
+* `append` operation added
+
 ## v0.9.0
 
 * Argments are no longer passed to the `call` method, but to the

data/README.md

@@ -276,6 +276,15 @@ The `key` itself is also passed to the block for the case that you need to
 customize the reduce calculation according to the value of the key. However,
 most of the time, this is not neccessary and the key can simply be ignored.
 
+* `append`: Appends the results of 2 jobs, such that all key-value pairs
+  of both jobs will be in the result. `append` does not accept any block.
+
+```ruby
+  job.append(other_job, worker: MyKrapsWorker, jobs: 8)
+```
+Please note that the partitioners and the number of partitions must match for
+the jobs to be appended.
+
 * `combine`: Combines the results of 2 jobs by combining every key available
   in the current job result with the corresponding key from the passed job
   result. When the passed job result does not have the corresponding key,

data/lib/kraps/actions.rb

@@ -6,7 +6,8 @@ module Kraps
       MAP_PARTITIONS = "map_partitions",
       REDUCE = "reduce",
       COMBINE = "combine",
-      EACH_PARTITION = "each_partition"
+      EACH_PARTITION = "each_partition",
+      APPEND = "append"
     ]
   end
 end

data/lib/kraps/job.rb

@@ -103,6 +103,24 @@ module Kraps
       end
     end
 
+    def append(other_job, jobs: nil, worker: @worker, before: nil, &block)
+      fresh.tap do |job|
+        job.instance_eval do
+          @steps << Step.new(
+            action: Actions::APPEND,
+            jobs: [jobs, @partitions].compact.min,
+            partitions: @partitions,
+            partitioner: @partitioner,
+            worker: worker,
+            before: before,
+            block: block,
+            dependency: other_job,
+            options: { append_step_index: other_job.steps.size - 1 }
+          )
+        end
+      end
+    end
+
     def each_partition(jobs: nil, worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do

data/lib/kraps/runner.rb

@@ -91,6 +91,21 @@ module Kraps
         Frame.new(token: token, partitions: @step.partitions)
       end
 
+      def perform_append
+        append_job = @step.dependency
+        append_step = append_job.steps[@step.options[:append_step_index]]
+
+        raise(IncompatibleFrame, "Incompatible number of partitions") if append_step.partitions != @step.partitions
+
+        enum = (0...@frame.partitions).map do |partition|
+          { partition: partition, append_frame: append_step.frame.to_h }
+        end
+
+        token = push_and_wait(job_count: @step.jobs, enum: enum)
+
+        Frame.new(token: token, partitions: @step.partitions)
+      end
+
       def perform_each_partition
         enum = (0...@frame.partitions).map { |partition| { partition: partition } }
         push_and_wait(job_count: @step.jobs, enum: enum)

data/lib/kraps/version.rb

@@ -1,3 +1,3 @@
 module Kraps
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/lib/kraps/worker.rb

@@ -86,6 +86,57 @@ module Kraps
       temp_paths&.delete
     end
 
+    def perform_append(payload)
+      temp_paths1 = download_all(token: @args["frame"]["token"], partition: payload["partition"])
+      temp_paths2 = download_all(token: payload["append_frame"]["token"], partition: payload["partition"])
+
+      implementation = Object.new
+      implementation.define_singleton_method(:map) do |key, value, &block|
+        block.call(key, value)
+      end
+
+      subsequent_step = next_step
+
+      if subsequent_step&.action == Actions::REDUCE
+        implementation.define_singleton_method(:reduce) do |key, value1, value2|
+          subsequent_step.block.call(key, value1, value2)
+        end
+      end
+
+      mapper = MapReduce::Mapper.new(implementation, partitioner: partitioner, memory_limit: @memory_limit)
+
+      temp_paths1.each do |temp_path|
+        File.open(temp_path.path) do |stream|
+          stream.each_line do |line|
+            key, value = JSON.parse(line)
+
+            mapper.map(key, value)
+          end
+        end
+      end
+
+      temp_paths2.each do |temp_path|
+        File.open(temp_path.path) do |stream|
+          stream.each_line do |line|
+            key, value = JSON.parse(line)
+
+            mapper.map(key, value)
+          end
+        end
+      end
+
+      mapper.shuffle(chunk_limit: @chunk_limit) do |partitions|
+        Parallelizer.each(partitions.to_a, @concurrency) do |partition, path|
+          File.open(path) do |stream|
+            Kraps.driver.store(Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{payload["partition"]}.json"), stream)
+          end
+        end
+      end
+    ensure
+      temp_paths1&.delete
+      temp_paths2&.delete
+    end
+
     def perform_map_partitions(payload)
       temp_paths = download_all(token: @args["frame"]["token"], partition: payload["partition"])
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kraps
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-13 00:00:00.000000000 Z
+date: 2024-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attachie