kraps 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19635ced3e745d44313ed3bc416ef73eb555134c591725f4dab7b38208e21393
-  data.tar.gz: bb7f679c7e2cd053744d1c7d857629de1912a305880d0538bdab418de3861ba1
+  metadata.gz: d261c779e82209152e26decbc6c5a6c5c5ddb0fb40803884383617635727d3b2
+  data.tar.gz: 1b9c6fa8db7a7811cbac5a7a5db518e1f3ee75df583521b64417341e830425f4
 SHA512:
-  metadata.gz: 91273ba54ea33c6d5cb1b4f335ad8039c35601953fdf1e6b9b2ac3117ceb25d81e9be569e5c8b70deda22b53a4c72f04d60ffb4b251badf5c5a64d13d399f36c
-  data.tar.gz: 4e563257fcba0c9f457b363da4f43d000ee239a5179e2f965071bb3df27e362cdf7bc9950a1624520cb862acfb0fde89e96b0a4987d79a93524230c4b84619cd
+  metadata.gz: dcb05139042149be087b1a2c7f14a31cd5e28dedb1517aca83299f63b90046e4d05e0ab19dfaeede329e784880623abda19675252cdeaad04f8ccd87249afde5
+  data.tar.gz: 10fd07c322c659ae21a682832eba30416c830f9d2146af685d69168ad5137045ef4268c0a43cee4e879bb875edf900ca740bbe4cbfe8b91b34ad3df40763bce0

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## v0.8.0
+
+* Use number of partitions of previous step for `jobs` option by default
+* Changed `combine` to receive a `collector`
+* Added mandatory `concurrency` argument to `load`
+
 ## v0.7.0
 
 * Added a `jobs` option to the actions to limit the concurrency

data/README.md

@@ -232,10 +232,11 @@ return the new key-value pair, but the `collector` must be used instead.
 The `jobs` argument can be useful when you need to access an external data
 source, like a relational database and you want to limit the number of workers
 accessing the store concurrently to avoid overloading it. If you don't specify
-it, it will be identical to the number of partitions. It is recommended to only
-use it for steps where you need to throttle the concurrency, because it will of
-course slow down the processing. The `jobs` argument only applies to the
-current step. The following steps don't inherit the argument, but reset it.
+it, it will be identical to the number of partitions of the previous step. It
+is recommended to only use it for steps where you need to throttle the
+concurrency, because it will of course slow down the processing. The `jobs`
+argument only applies to the current step. The following steps don't inherit
+the argument, but reset it.
 
 * `map_partitions`: Maps the key value pairs to other key value pairs, but the
   block receives all data of each partition as an enumerable and sorted by key.
@@ -273,8 +274,8 @@ most of the time, this is not neccessary and the key can simply be ignored.
   passed job result are completely omitted.
 
 ```ruby
-  job.combine(other_job, worker: MyKrapsWorker, jobs: 8) do |key, value1, value2|
-    (value1 || {}).merge(value2 || {})
+  job.combine(other_job, worker: MyKrapsWorker, jobs: 8) do |key, value1, value2, collector|
+    collector.call(key, (value1 || {}).merge(value2 || {}))
   end
 ```
 
@@ -316,10 +317,12 @@ It creates a folder for every partition and stores one or more chunks in there.
 * `load`: Loads the previously dumped data
 
 ```ruby
-job.load(prefix: "path/to/dump", partitions: 32, partitioner: Kraps::HashPartitioner.new, worker: MyKrapsWorker)
+job.load(prefix: "path/to/dump", partitions: 32, concurrency: 8, partitioner: Kraps::HashPartitioner.new, worker: MyKrapsWorker)
 ```
 
-The number of partitions and the partitioner must be specified.
+The number of partitions, the partitioner and concurrency must be specified.
+The concurrency specifies the number of threads used for downloading chunks in
+parallel.
 
 Please note that every API method accepts a `before` callable:
 

data/lib/kraps/downloader.rb

@@ -0,0 +1,19 @@
+module Kraps
+  class Downloader
+    def self.download_all(prefix:, concurrency:)
+      temp_paths = TempPaths.new
+
+      files = Kraps.driver.list(prefix: prefix).sort
+
+      temp_paths_index = files.each_with_object({}) do |file, hash|
+        hash[file] = temp_paths.add
+      end
+
+      Parallelizer.each(files, concurrency) do |file|
+        Kraps.driver.download(file, temp_paths_index[file].path)
+      end
+
+      temp_paths
+    end
+  end
+end

data/lib/kraps/job.rb

@@ -30,12 +30,14 @@ module Kraps
     def map(partitions: nil, partitioner: nil, jobs: nil, worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do
+          jobs = [jobs, @partitions].compact.min
+
           @partitions = partitions if partitions
           @partitioner = partitioner if partitioner
 
           @steps << Step.new(
             action: Actions::MAP,
-            jobs: [jobs, @partitions].compact.min,
+            jobs: jobs,
             partitions: @partitions,
             partitioner: @partitioner,
             worker: worker,
@@ -49,12 +51,14 @@ module Kraps
     def map_partitions(partitions: nil, partitioner: nil, jobs: nil, worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do
+          jobs = [jobs, @partitions].compact.min
+
           @partitions = partitions if partitions
           @partitioner = partitioner if partitioner
 
           @steps << Step.new(
             action: Actions::MAP_PARTITIONS,
-            jobs: [jobs, @partitions].compact.min,
+            jobs: jobs,
             partitions: @partitions,
             partitioner: @partitioner,
             worker: worker,
@@ -135,7 +139,7 @@ module Kraps
       end
     end
 
-    def load(prefix:, partitions:, partitioner:, worker: @worker)
+    def load(prefix:, partitions:, partitioner:, concurrency:, worker: @worker)
       job = parallelize(partitions: partitions, partitioner: proc { |key, _| key }, worker: worker) do |collector|
         (0...partitions).each do |partition|
           collector.call(partition)
@@ -143,20 +147,19 @@ module Kraps
       end
 
       job.map_partitions(partitioner: partitioner, worker: worker) do |partition, _, collector|
-        tempfile = Tempfile.new
-
-        path = File.join(prefix, partition.to_s, "chunk.json")
-        next unless Kraps.driver.exists?(path)
+        temp_paths = Downloader.download_all(prefix: File.join(prefix, partition.to_s, "/"), concurrency: concurrency)
 
-        Kraps.driver.download(path, tempfile.path)
+        temp_paths.each do |temp_path|
+          File.open(temp_path.path) do |stream|
+            stream.each_line do |line|
+              key, value = JSON.parse(line)
 
-        tempfile.each_line do |line|
-          key, value = JSON.parse(line)
-
-          collector.call(key, value)
+              collector.call(key, value)
+            end
+          end
         end
       ensure
-        tempfile&.close(true)
+        temp_paths&.delete
       end
     end
 

data/lib/kraps/runner.rb

@@ -100,7 +100,7 @@ module Kraps
 
       def push_and_wait(enum:, job_count: nil)
         redis_queue = RedisQueue.new(redis: Kraps.redis, token: SecureRandom.hex, namespace: Kraps.namespace, ttl: Kraps.job_ttl)
-        progress_bar = build_progress_bar("#{@klass}: job #{@job_index + 1}/#{@jobs.size}, step #{@step_index + 1}/#{@job.steps.size}, token #{redis_queue.token}, %a, %c/%C (%p%) => #{@step.action}")
+        progress_bar = build_progress_bar("#{@klass}: job #{@job_index + 1}/#{@jobs.size}, step #{@step_index + 1}/#{@job.steps.size}, #{@step.jobs || "?"} jobs, token #{redis_queue.token}, %a, %c/%C (%p%) => #{@step.action}")
 
         total = 0
 

data/lib/kraps/version.rb

@@ -1,3 +1,3 @@
 module Kraps
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/kraps/worker.rb

@@ -157,7 +157,7 @@ module Kraps
       implementation = Object.new
       implementation.define_singleton_method(:map) do |&block|
         combine_method.call(enum1, enum2) do |key, value1, value2|
-          block.call(key, current_step.block.call(key, value1, value2))
+          current_step.block.call(key, value1, value2, block)
         end
       end
 
@@ -270,19 +270,7 @@ module Kraps
     end
 
     def download_all(token:, partition:)
-      temp_paths = TempPaths.new
-
-      files = Kraps.driver.list(prefix: Kraps.driver.with_prefix("#{token}/#{partition}/")).sort
-
-      temp_paths_index = files.each_with_object({}) do |file, hash|
-        hash[file] = temp_paths.add
-      end
-
-      Parallelizer.each(files, @concurrency) do |file|
-        Kraps.driver.download(file, temp_paths_index[file].path)
-      end
-
-      temp_paths
+      Downloader.download_all(prefix: Kraps.driver.with_prefix("#{token}/#{partition}/"), concurrency: @concurrency)
     end
 
     def jobs

data/lib/kraps.rb

@@ -19,6 +19,7 @@ require_relative "kraps/runner"
 require_relative "kraps/step"
 require_relative "kraps/frame"
 require_relative "kraps/worker"
+require_relative "kraps/downloader"
 
 module Kraps
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kraps
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-02 00:00:00.000000000 Z
+date: 2023-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attachie
@@ -128,6 +128,7 @@ files:
 - docker-compose.yml
 - lib/kraps.rb
 - lib/kraps/actions.rb
+- lib/kraps/downloader.rb
 - lib/kraps/drivers.rb
 - lib/kraps/frame.rb
 - lib/kraps/hash_partitioner.rb