kraps 0.8.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d261c779e82209152e26decbc6c5a6c5c5ddb0fb40803884383617635727d3b2
-  data.tar.gz: 1b9c6fa8db7a7811cbac5a7a5db518e1f3ee75df583521b64417341e830425f4
+  metadata.gz: 6ac0098a3c2f8acf49552f5b72621619345f9b59c83edd58368d7a999605f817
+  data.tar.gz: 3d522dd1149b57c9dce596ea4b38aa67ba5f2bc18a96bbc2840e80096e9eb192
 SHA512:
-  metadata.gz: dcb05139042149be087b1a2c7f14a31cd5e28dedb1517aca83299f63b90046e4d05e0ab19dfaeede329e784880623abda19675252cdeaad04f8ccd87249afde5
-  data.tar.gz: 10fd07c322c659ae21a682832eba30416c830f9d2146af685d69168ad5137045ef4268c0a43cee4e879bb875edf900ca740bbe4cbfe8b91b34ad3df40763bce0
+  metadata.gz: e859a2ecd0e635a067bde3cf3d51f76dc64702d91975342c590a43370d5a8a49da5c2272b6806550420537c74b3c7129ee0c76f58ff42ef0b154d7e8d0904417
+  data.tar.gz: 8e93aa69360b48b46f96a94cc2dd49d841b72fc858688c93071d46b99b86d50bf3b0758dd7f071cb50fd6ddc13cf532ed00d1ed1e4eee277d064568a11931f22

data/.rubocop.yml

@@ -16,6 +16,12 @@ Lint/UnreachableLoop:
 Metrics/BlockLength:
   Enabled: false
 
+Style/HashEachMethods:
+  Enabled: false
+
+Style/ZeroLengthPredicate:
+  Enabled: false
+
 Gemspec/RequiredRubyVersion:
   Enabled: false
 
@@ -48,6 +54,9 @@ Style/StringLiteralsInInterpolation:
   Enabled: true
   EnforcedStyle: double_quotes
 
+Style/RedundantSelfAssignment:
+  Enabled: false
+
 Layout/LineLength:
   Max: 250
 

data/CHANGELOG.md

@@ -1,5 +1,39 @@
 # CHANGELOG
 
+## v0.10.0
+
+* `append` operation added
+
+## v0.9.0
+
+* Argments are no longer passed to the `call` method, but to the
+  initializer instead
+
+Before:
+
+```ruby
+class MyJob
+  def call(arg1, arg2)
+    # ...
+  end
+end
+```
+
+After:
+
+```ruby
+class MyJob
+  def initializer(arg1, arg2)
+    @arg1 = arg1
+    @arg2 = arg2
+  end
+
+  def call
+    # ...
+  end
+end
+```
+
 ## v0.8.0
 
 * Use number of partitions of previous step for `jobs` option by default

data/Gemfile

@@ -3,6 +3,7 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in kraps.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
-
-gem "rspec", "~> 3.0"
+gem "bundler"
+gem "rake"
+gem "rspec"
+gem "rubocop"

data/README.md

@@ -38,16 +38,21 @@ Kraps.configure(
 
 Afterwards, create a job class, which tells Kraps what your job should do.
 Therefore, you create some class with a `call` method, and optionally some
-arguments. Let's create a simple job, which reads search log files to analyze
-how often search queries have been searched:
+arguments passed to its initializer. Let's create a simple job, which reads
+search log files to analyze how often search queries have been searched:
 
 ```ruby
 class SearchLogCounter
-  def call(start_date:, end_date:)
+  def initialize(start_date:, end_date:)
+    @start_date = start_date
+    @end_date = end_date
+  end
+
+  def call
     job = Kraps::Job.new(worker: MyKrapsWorker)
 
     job = job.parallelize(partitions: 128) do |collector|
-      (Date.parse(start_date)..Date.parse(end_date)).each do |date|
+      (Date.parse(@start_date)..Date.parse(@end_date)).each do |date|
         collector.call(date.to_s)
       end
     end
@@ -214,6 +219,10 @@ job.parallelize(partitions: 128, partitioner: partitioner, worker: MyKrapsWorker
 end
 ```
 
+Please note, that `parallelize` itself is not parallelized but rather
+parallelizes the data you feed into Kraps within `parallelize` by splitting it
+into the number of `partitions` specified.
+
 The block must use the collector to feed Kraps with individual items. The
 items are used as keys and the values are set to `nil`.
 
@@ -267,6 +276,15 @@ The `key` itself is also passed to the block for the case that you need to
 customize the reduce calculation according to the value of the key. However,
 most of the time, this is not neccessary and the key can simply be ignored.
 
+* `append`: Appends the results of 2 jobs, such that all key-value pairs
+  of both jobs will be in the result. `append` does not accept any block.
+
+```ruby
+  job.append(other_job, worker: MyKrapsWorker, jobs: 8)
+```
+Please note that the partitioners and the number of partitions must match for
+the jobs to be appended.
+
 * `combine`: Combines the results of 2 jobs by combining every key available
   in the current job result with the corresponding key from the passed job
   result. When the passed job result does not have the corresponding key,
@@ -345,11 +363,16 @@ of searches made:
 
 ```ruby
 class SearchLogCounter
-  def call(start_date:, end_date:)
+  def initialize(start_date:, end_date:)
+    @start_date = start_date
+    @end_date = end_date
+  end
+
+  def call
     count_job = Kraps::Job.new(worker: SomeBackgroundWorker)
 
     count_job = count_job.parallelize(partitions: 128) do |collector|
-      (Date.parse(start_date)..Date.parse(end_date)).each do |date|
+      (Date.parse(@start_date)..Date.parse(@end_date)).each do |date|
         collector.call(date.to_s)
       end
     end

data/lib/kraps/actions.rb

@@ -6,7 +6,8 @@ module Kraps
       MAP_PARTITIONS = "map_partitions",
       REDUCE = "reduce",
       COMBINE = "combine",
-      EACH_PARTITION = "each_partition"
+      EACH_PARTITION = "each_partition",
+      APPEND = "append"
     ]
   end
 end

data/lib/kraps/job.rb

@@ -103,6 +103,24 @@ module Kraps
       end
     end
 
+    def append(other_job, jobs: nil, worker: @worker, before: nil, &block)
+      fresh.tap do |job|
+        job.instance_eval do
+          @steps << Step.new(
+            action: Actions::APPEND,
+            jobs: [jobs, @partitions].compact.min,
+            partitions: @partitions,
+            partitioner: @partitioner,
+            worker: worker,
+            before: before,
+            block: block,
+            dependency: other_job,
+            options: { append_step_index: other_job.steps.size - 1 }
+          )
+        end
+      end
+    end
+
     def each_partition(jobs: nil, worker: @worker, before: nil, &block)
       fresh.tap do |job|
         job.instance_eval do

data/lib/kraps/runner.rb

@@ -5,7 +5,7 @@ module Kraps
     end
 
     def call(*args, **kwargs)
-      JobResolver.new.call(@klass.new.call(*args, **kwargs)).tap do |jobs|
+      JobResolver.new.call(@klass.new(*args, **kwargs).call).tap do |jobs|
         jobs.each_with_index do |job, job_index|
           job.steps.each_with_index.inject(nil) do |frame, (_, step_index)|
             StepRunner.new(
@@ -91,6 +91,21 @@ module Kraps
         Frame.new(token: token, partitions: @step.partitions)
       end
 
+      def perform_append
+        append_job = @step.dependency
+        append_step = append_job.steps[@step.options[:append_step_index]]
+
+        raise(IncompatibleFrame, "Incompatible number of partitions") if append_step.partitions != @step.partitions
+
+        enum = (0...@frame.partitions).map do |partition|
+          { partition: partition, append_frame: append_step.frame.to_h }
+        end
+
+        token = push_and_wait(job_count: @step.jobs, enum: enum)
+
+        Frame.new(token: token, partitions: @step.partitions)
+      end
+
       def perform_each_partition
         enum = (0...@frame.partitions).map { |partition| { partition: partition } }
         push_and_wait(job_count: @step.jobs, enum: enum)

data/lib/kraps/version.rb

@@ -1,3 +1,3 @@
 module Kraps
-  VERSION = "0.8.0"
+  VERSION = "0.10.0"
 end

data/lib/kraps/worker.rb

@@ -86,6 +86,57 @@ module Kraps
       temp_paths&.delete
     end
 
+    def perform_append(payload)
+      temp_paths1 = download_all(token: @args["frame"]["token"], partition: payload["partition"])
+      temp_paths2 = download_all(token: payload["append_frame"]["token"], partition: payload["partition"])
+
+      implementation = Object.new
+      implementation.define_singleton_method(:map) do |key, value, &block|
+        block.call(key, value)
+      end
+
+      subsequent_step = next_step
+
+      if subsequent_step&.action == Actions::REDUCE
+        implementation.define_singleton_method(:reduce) do |key, value1, value2|
+          subsequent_step.block.call(key, value1, value2)
+        end
+      end
+
+      mapper = MapReduce::Mapper.new(implementation, partitioner: partitioner, memory_limit: @memory_limit)
+
+      temp_paths1.each do |temp_path|
+        File.open(temp_path.path) do |stream|
+          stream.each_line do |line|
+            key, value = JSON.parse(line)
+
+            mapper.map(key, value)
+          end
+        end
+      end
+
+      temp_paths2.each do |temp_path|
+        File.open(temp_path.path) do |stream|
+          stream.each_line do |line|
+            key, value = JSON.parse(line)
+
+            mapper.map(key, value)
+          end
+        end
+      end
+
+      mapper.shuffle(chunk_limit: @chunk_limit) do |partitions|
+        Parallelizer.each(partitions.to_a, @concurrency) do |partition, path|
+          File.open(path) do |stream|
+            Kraps.driver.store(Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{payload["partition"]}.json"), stream)
+          end
+        end
+      end
+    ensure
+      temp_paths1&.delete
+      temp_paths2&.delete
+    end
+
     def perform_map_partitions(payload)
       temp_paths = download_all(token: @args["frame"]["token"], partition: payload["partition"])
 
@@ -274,7 +325,7 @@ module Kraps
     end
 
     def jobs
-      @jobs ||= JobResolver.new.call(@args["klass"].constantize.new.call(*@args["args"], **@args["kwargs"].transform_keys(&:to_sym)))
+      @jobs ||= JobResolver.new.call(@args["klass"].constantize.new(*@args["args"], **@args["kwargs"].transform_keys(&:to_sym)).call)
     end
 
     def job

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kraps
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-13 00:00:00.000000000 Z
+date: 2024-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attachie
@@ -66,48 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Kraps allows to process and perform calculations on very large datasets
   in parallel
 email:
@@ -121,7 +79,6 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile

data/Gemfile.lock

@@ -1,108 +0,0 @@
-PATH
-  remote: .
-  specs:
-    kraps (0.7.0)
-      attachie
-      map-reduce-ruby (>= 3.0.0)
-      redis
-      ruby-progressbar
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (7.0.4)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-    ast (2.4.2)
-    attachie (1.2.0)
-      activesupport
-      aws-sdk-s3
-      connection_pool
-      mime-types
-    aws-eventstream (1.2.0)
-    aws-partitions (1.657.0)
-    aws-sdk-core (3.166.0)
-      aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.651.0)
-      aws-sigv4 (~> 1.5)
-      jmespath (~> 1, >= 1.6.1)
-    aws-sdk-kms (1.59.0)
-      aws-sdk-core (~> 3, >= 3.165.0)
-      aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.117.1)
-      aws-sdk-core (~> 3, >= 3.165.0)
-      aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.4)
-    aws-sigv4 (1.5.2)
-      aws-eventstream (~> 1, >= 1.0.2)
-    concurrent-ruby (1.1.10)
-    connection_pool (2.3.0)
-    diff-lcs (1.5.0)
-    i18n (1.12.0)
-      concurrent-ruby (~> 1.0)
-    jmespath (1.6.1)
-    json (2.6.2)
-    lazy_priority_queue (0.1.1)
-    map-reduce-ruby (3.0.0)
-      json
-      lazy_priority_queue
-    mime-types (3.4.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
-    minitest (5.16.3)
-    parallel (1.22.1)
-    parser (3.1.2.1)
-      ast (~> 2.4.1)
-    rainbow (3.1.1)
-    rake (13.0.6)
-    redis (5.0.5)
-      redis-client (>= 0.9.0)
-    redis-client (0.11.2)
-      connection_pool
-    regexp_parser (2.6.0)
-    rexml (3.2.5)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.0)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.38.0)
-      json (~> 2.3)
-      parallel (~> 1.10)
-      parser (>= 3.1.2.1)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.23.0, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.23.0)
-      parser (>= 3.1.1.0)
-    ruby-progressbar (1.11.0)
-    tzinfo (2.0.5)
-      concurrent-ruby (~> 1.0)
-    unicode-display_width (2.3.0)
-
-PLATFORMS
-  ruby
-  x86_64-linux
-
-DEPENDENCIES
-  bundler
-  kraps!
-  rake (~> 13.0)
-  rspec (~> 3.0)
-  rubocop
-
-BUNDLED WITH
-   2.3.24