fluent-plugin-gcloud-pubsub-custom 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fca9651c5d6e9f497579b5ee6ba6a42f2bb4e69a
-  data.tar.gz: d3a2a04f0e645a04a9828188715a4fab827cd4c6
+  metadata.gz: 1b951f60017ac6da9a7cab1d745b4d9f52ffafe1
+  data.tar.gz: eae153dca1b2b632c0bddfa196cd19949eff6aae
 SHA512:
-  metadata.gz: 14806e97d5234bf531850e890b53066e65deb222e9eb602d509102f09496d38a8082568ae228a9ff74cfb544323327529422de22a39d1ac5c5977ea6208e27ea
-  data.tar.gz: e874e411b67cfaac3e0f13635661206fd50a812104d2ac0bb2cf68eb3d17f3c74d9b05797af1a4b14d9bd4f646602823631f96641640e61f6a2e1ce053a42698
+  metadata.gz: b58c8a1285d86cb1f961aaca2042c9f0c6e9e54aef2d893149d6a3f53c825a2200cdcce574de68ffc38a4b6e46029ce87581ce72595e475cf2d9d5328be90e66
+  data.tar.gz: 73ce9c108180b311d839b87f68ba8a0309cf2d681cabbf6a4f5beb2503450e41a7cdcc147c2c7bd5d4b89fa40ba5c8743c382013072cf9c65e1b310d4a96142e

data/CHANGELOG.md

@@ -1,9 +1,15 @@
 ## ChangeLog
 
+### Release 0.3.2 - 2016/11/13
+
+- Add plugin param desc
+- Input plugin
+  - Improve handling to acknowledge messages
+
 ### Release 0.3.1 - 2016/11/03
 
 - Output plugin
-- Improve error handling
+  - Improve error handling
 
 ### Release 0.3.0 - 2016/10/30
 

data/README.md

@@ -27,6 +27,12 @@ Install by gem:
 $ gem install fluent-plugin-gcloud-pubsub-custom
 ```
 
+**Caution**
+
+This plugin doesn't work in [td-agent](http://docs.fluentd.org/articles/install-by-rpm).
+
+Please use in [Fluentd installed by gem](http://docs.fluentd.org/articles/install-by-gem).
+
 ## Configuration
 
 ### Publish messages
@@ -51,7 +57,7 @@ Use `gcloud_pubsub` output plugin.
 ```
 
 - `project` (optional)
-  - Set your GCP project
+  - Set your GCP project.
   - Running fluentd on GCP, you don't have to specify.
   - You can also use environment variable such as `GCLOUD_PROJECT`.
 - `key` (optional)

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.3.1"
+  gem.version     = "0.3.2"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -10,20 +10,43 @@ module Fluent
   class GcloudPubSubInput < Input
     Fluent::Plugin.register_input('gcloud_pubsub', self)
 
+    class << self
+      unless method_defined?(:desc)
+        def desc(description)
+        end
+      end
+    end
+
+    class FailedParseError < StandardError
+    end
+
+    desc 'Set tag of messages.'
     config_param :tag,                :string
+    desc 'Set your GCP project.'
     config_param :project,            :string,  default: nil
+    desc 'Set your credential file path.'
     config_param :key,                :string,  default: nil
+    desc 'Set topic name to pull.'
     config_param :topic,              :string
+    desc 'Set subscription name to pull.'
     config_param :subscription,       :string
+    desc 'Pulling messages by intervals of specified seconds.'
     config_param :pull_interval,      :float,   default: 5.0
+    desc 'Max messages pulling at once.'
     config_param :max_messages,       :integer, default: 100
+    desc 'Setting `true`, keepalive connection to wait for new messages.'
     config_param :return_immediately, :bool,    default: true
+    desc 'Set number of threads to pull messages.'
     config_param :pull_threads,       :integer, default: 1
+    desc 'Set input format.'
     config_param :format,             :string,  default: 'json'
     # for HTTP RPC
+    desc 'If `true` is specified, HTTP RPC to stop or start pulling message is enabled.'
     config_param :enable_rpc,         :bool,    default: false
-    config_param :rpc_bind,               :string,  default: '0.0.0.0'
-    config_param :rpc_port,               :integer, default: 24680
+    desc 'Bind IP address for HTTP RPC.'
+    config_param :rpc_bind,           :string,  default: '0.0.0.0'
+    desc 'Port for HTTP RPC.'
+    config_param :rpc_port,           :integer, default: 24680
 
     unless method_defined?(:log)
       define_method("log") { $log }
@@ -163,44 +186,27 @@ module Fluent
         return
       end
 
-      es = parse_messages(messages)
-      if es.empty?
-        log.warn "#{messages.length} message(s) are pulled, but no messages are parsed"
-        return
-      end
-
-      begin
-        router.emit_stream(@tag, es)
-      rescue
-        # ignore errors. Engine shows logs and backtraces.
-      end
+      process messages
       @subscriber.acknowledge messages
+
       log.debug "#{messages.length} message(s) processed"
     rescue => ex
       log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
       log.error_backtrace ex.backtrace
     end
 
-    def parse_messages(messages)
+    def process(messages)
       es = MultiEventStream.new
       messages.each do |m|
-        convert_line_to_event(m.message.data, es)
-      end
-      es
-    end
-
-    def convert_line_to_event(line, es)
-      line = line.chomp  # remove \n
-      @parser.parse(line) { |time, record|
-        if time && record
-          es.add(time, record)
-        else
-          log.warn "pattern not match: #{line.inspect}"
+        @parser.parse(m.message.data.chomp) do |time, record|
+          if time && record
+            es.add(time, record)
+          else
+            raise FailedParseError.new "pattern not match: #{line.inspect}"
+          end
         end
-      }
-    rescue => ex
-      log.warn line.dump, error_message: ex.to_s, error_class: ex.class.to_s
-      log.warn_backtrace ex.backtrace
+      end
+      router.emit_stream(@tag, es)
     end
   end
 end

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -6,12 +6,26 @@ module Fluent
   class GcloudPubSubOutput < BufferedOutput
     Fluent::Plugin.register_output('gcloud_pubsub', self)
 
+    class << self
+      unless method_defined?(:desc)
+        def desc(description)
+        end
+      end
+    end
+
+    desc 'Set your GCP project.'
     config_param :project,            :string,  :default => nil
+    desc 'Set your credential file path.'
     config_param :key,                :string,  :default => nil
+    desc 'Set topic name to publish.'
     config_param :topic,              :string
+    desc "If set to `true`, specified topic will be created when it doesn't exist."
     config_param :autocreate_topic,   :bool,    :default => false
+    desc 'Publishing messages count per request to Cloud Pub/Sub.'
     config_param :max_messages,       :integer, :default => 1000
+    desc 'Publishing messages bytesize per request to Cloud Pub/Sub.'
     config_param :max_total_size,     :integer, :default => 9800000  # 9.8MB
+    desc 'Set output format.'
     config_param :format,             :string,  :default => 'json'
 
     unless method_defined?(:log)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-03 00:00:00.000000000 Z
+date: 2016-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd