embulk-input-buildkite 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8181f9bd9a42836709a5deb133f20a93d4465e1087eaaa2701f77af0f9c30d86
-  data.tar.gz: 4dbcd67665f3659150e92b1ea04adb42df2bbff9d9494b713e3dec8f4e96b388
+  metadata.gz: 5ad00bb2939704ea96a218b7825190027e7fe47c44cc9a5fad3128805e728360
+  data.tar.gz: afcc34e688f3bee95aeb78d5d4d0d29cbac22ab372b12cdd034cf29a0b0cfc6d
 SHA512:
-  metadata.gz: 58ecb609e666d22b57fe8f4197cc0a4cb6a3fbc782c4e88dd97bccbe342e771cf1731a644d69a7fb81bfdb4b73fb6b531576a5fd2716991a17dc24a6a3754fdf
-  data.tar.gz: 3375bf15b396417cd9890505de0829eee94317ebf67c890243e19f96130107d43ec9d5b4d5c4b76ec61d5a0643f7caeb7c5bcaa2f51afda6ec7fce7ce41b53e0
+  metadata.gz: e4a0161cb327a962851529d364f426672bbd277e147eacdc4974ffc22574519b2440a414c98936b12452242f542cb5184e5597af11eb0a95d62cdc69fb0290cd
+  data.tar.gz: 067a7064167107a9999d348ddc711fc6764102751d0363b8f8261e802629c49be7797fac331b889b14087c3b135db487c348ddac841270a9e78353d566a23af6

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.4.0
+
+- Add retry
+- Add artifact_download_concurrency option
+
 # 0.3.0
 
 - Add artifacts

data/README.md

@@ -15,6 +15,7 @@ Fetch Buildkite build results
 - **pipeline_slug**: Target pipeline name like `rails` (string, required)
 - **build_nums**: Build numbers (Array of integer, required)
 - **token**: Buildkite API token which can be found on https://buildkite.com/user/api-access-tokens (string, required)
+- **artifact_download_concurrency**: Artifact download concurrency (integer, optional, default: 10)
 
 ## Example
 

data/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "2.4"
 services:
   embulk:
     build: .
@@ -12,6 +12,7 @@ services:
     environment:
       BUNDLE_PATH: /vendor/bundle
     env_file: app.env
+    mem_limit: 512M
 volumes:
   home:
   vendor:

data/embulk-input-buildkite.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-buildkite"
-  spec.version       = "0.3.0"
+  spec.version       = "0.4.0"
   spec.authors       = ["Fumiaki MATSUSHIMA"]
   spec.summary       = "Buildkite input plugin for Embulk"
   spec.description   = "Loads records from Buildkite."

data/lib/embulk/input/buildkite.rb

@@ -4,13 +4,17 @@ module Embulk
   module Input
 
     class Buildkite < InputPlugin
+      MAX_RETRY = 5
+
       class Logger
         def initialize(embulk_logger)
           @embulk_logger = embulk_logger
         end
 
-        def info(message)
-          @embulk_logger.info("embulk-input-buildkite: #{message}")
+        %w(info warn error).each do |m|
+          define_method(m) do |message|
+            @embulk_logger.send(m, "embulk-input-buildkite: #{message}")
+          end
         end
       end
 
@@ -22,7 +26,8 @@ module Embulk
           "org_slug" => config.param("org_slug", :string),
           "pipeline_slug" => config.param("pipeline_slug", :string),
           "build_nums" => config.param("build_nums", :array),
-          "token" => config.param("token", :string, nil),
+          "token" => config.param("token", :string),
+          "artifact_download_concurrency" => config.param("artifact_download_concurrency", :integer, default: 10),
         }
 
         columns = [
@@ -63,14 +68,25 @@ module Embulk
         task['build_nums'].each do |build_num|
           logger.info("Start build_num:[#{build_num}]")
 
-          build = client.fetch_build(number: build_num)
+          build = with_retry { client.fetch_build(number: build_num) }
           build[:jobs].each do |job|
             logger.info("Start Start job_id:[#{job[:id]}]")
-            log = client.fetch_log(build_number: job[:build_number], job_id: job[:id])
-            artifacts = client.fetch_artifacts(build_number: job[:build_number], job_id: job[:id])
-            artifacts.each do |artifact|
-              artifact.merge!(body: client.fetch_artifact(build_number: job[:build_number], job_id: job[:id], artifact_id: artifact[:id]))
+            log = with_retry { client.fetch_log(build_number: job[:build_number], job_id: job[:id]) }
+            artifacts = with_retry { client.fetch_artifacts(build_number: job[:build_number], job_id: job[:id]) }
+
+            queue = Queue.new
+            artifacts.each {|a| queue.push(a) }
+            workers = Array.new(task['artifact_download_concurrency']) do
+              Thread.new do
+                begin
+                  while artifact = queue.pop(true)
+                    artifact[:body] = with_retry { client.fetch_artifact(build_number: job[:build_number], job_id: job[:id], artifact_id: artifact[:id]) }
+                  end
+                rescue ThreadError
+                end
+              end
             end
+            workers.each(&:join)
 
             page_builder.add([
               job[:id],
@@ -101,6 +117,24 @@ module Embulk
       def logger
         @logger ||= Logger.new(Embulk.logger)
       end
+
+      def with_retry(&block)
+        retries = 0
+        begin
+          yield
+        rescue => e
+          sleep retries
+
+          if retries < MAX_RETRY
+            retries += 1
+            logger.warn("retry ##{retries}, #{e.message}")
+            retry
+          else
+            logger.error("retry exhausted ##{retries}, #{e.message}")
+            raise e
+          end
+        end
+      end
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-buildkite
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Fumiaki MATSUSHIMA