fluent-plugin-cloud-pubsub 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f2e73ecd93f477db47b6b65460ae859d9866d488
+  data.tar.gz: 153f860d1f0f481a3b140042c831ad8a8976ecf8
+SHA512:
+  metadata.gz: bb3779c706f2cbaebb60eca80c9ed251aafe2bd3da4322a84f8148731351023ed963fd9420813976586b542666e73f1343ed8ccb9e75fcad194a293fe9633223
+  data.tar.gz: 615977a94d7de3a796ecf8697f09d3bfd4b5f6c6d9a87ef37e2a94ebcccbfa5598bdb5b323205c9eb76297a7cecb20cc627ce4d324e98032a15c96442742da18

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Keiji Yoshida
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,2 @@
+# fluent-plugin-cloud-pubsub
+fluent-plugin-cloud-pubsub

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-gcloud-pubsub.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-cloud-pubsub"
+  gem.description = "Fluentd plugin for sending data to Cloud Pub/Sub"
+  gem.license     = "MIT"
+  gem.homepage    = "https://github.com/yosssi/fluent-plugin-cloud-pubsub"
+  gem.summary     = gem.description
+  gem.version     = "0.0.1"
+  gem.authors     = ["Keiji Yoshida"]
+  gem.email       = "yoshida.keiji.84@gmail.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "fluentd", "~> 0.12.0"
+  gem.add_runtime_dependency "gcloud", "~> 0.3.0"
+  gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
+
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
+end

data/lib/fluent/plugin/out_cloud_pubsub.rb

@@ -0,0 +1,74 @@
+require 'gcloud'
+
+module Fluent
+  class CloudPubSubOutput < BufferedOutput
+    MAX_REQ_SIZE = 10 * 1024 * 1024 # 10 MB
+    MAX_MSGS_PER_REQ = 1000
+
+    Plugin.register_output('cloud_pubsub', self)
+
+    config_param :project,          :string,  :default => nil
+    config_param :topic,            :string,  :default => nil
+    config_param :key,              :string,  :default => nil
+    config_param :max_req_size,     :integer, :default => MAX_REQ_SIZE
+    config_param :max_msgs_per_req, :integer, :default => MAX_MSGS_PER_REQ
+
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    def configure(conf)
+      super
+
+      raise Fluent::ConfigError, "'project' must be specified." unless @project
+      raise Fluent::ConfigError, "'topic' must be specified." unless @topic
+      raise Fluent::ConfigError, "'key' must be specified." unless @key
+    end
+
+    def start
+      super
+
+      pubsub = (Gcloud.new @project, @key).pubsub
+      @client = pubsub.topic @topic
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      msgs = []
+      msgs_size = 0
+
+      chunk.msgpack_each do |tag, time, record|
+        size = Yajl.dump(record).bytesize
+        if msgs.length > 0 && (msgs_size + size > max_req_size || msgs.length + 1 > max_msgs_per_req)
+          publish(msgs)
+          msgs = []
+          msgs_size = 0
+        end
+        msgs << record.to_json
+        msgs_size += size
+      end
+
+      if msgs.length > 0
+        publish(msgs)
+      end
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+  end
+
+  def publish(msgs)
+    @client.publish do |batch|
+      msgs.each do |m|
+        batch.publish m
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-cloud-pubsub
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Keiji Yoshida
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  name: gcloud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin for sending data to Cloud Pub/Sub
+email: yoshida.keiji.84@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-gcloud-pubsub.gemspec
+- lib/fluent/plugin/out_cloud_pubsub.rb
+homepage: https://github.com/yosssi/fluent-plugin-cloud-pubsub
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin for sending data to Cloud Pub/Sub
+test_files: []