fluent-plugin-gcloud-storage 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d305c8f71fafe1ec78eab6aca3abd92f210b9bc
+  data.tar.gz: 8be97348873b53a49ca2248418ff5141d9d4cd35
+SHA512:
+  metadata.gz: 7d6a4049c2aa0f9a1b78f65003459c16243de55320f09ccc0c4a691516cf30569a8ab12f508a3a5b3f4b4f519afb37b674d63f6b6c92c9d4936591ef689ad307
+  data.tar.gz: 7a6a8f29f5e010ea4a758e03f36346e35971ca8edfa8a57695d17d756ce76202a238ec87ad12edcc69ffba695ab502c781d616a56319dff28706763a3bfc1276

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.10.6

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-gcloud-storage.gemspec
+gemspec

data/README.md

@@ -0,0 +1,57 @@
+# fluent-plugin-gcloud-storage
+[![Build Status](https://travis-ci.org/trekdemo/fluent-plugin-gcloud-storage.svg?branch=master)](https://travis-ci.org/trekdemo/fluent-plugin-gcloud-storage)
+
+[Cloud Storage](https://cloud.google.com/storage/) Output plugin for [Fluentd](http://www.fluentd.org/) with [gcloud](https://googlecloudplatform.github.io/gcloud-ruby/) gem.
+
+Sponsored by [BIDMATH](http://bidmath.com)
+
+## Installation
+
+Please follow the [Plugin Management](http://docs.fluentd.org/articles/plugin-management) guide of fluentd.
+
+```ruby
+# If you're using fluentd
+fluent-gem install 'fluent-plugin-gcloud-storage'
+
+# If you're using td-agent
+td-agent-gem install 'fluent-plugin-gcloud-storage'
+```
+
+## Preparation
+
+- Create a project on Google Developer Console
+- Create a bucket within your project
+- Download your credential (json)
+
+## Configuration
+publish dummy json data like `{"message": "dummy", "value": 0}\n{"message": "dummy", "value": 1}\n ...`.
+
+```
+<source>
+  type dummy
+  tag example.publish
+  auto_increment_key value
+</source>
+
+<match example.publish>
+  type           gcloud_storage
+
+  key            <PATH YOUR KEY JSON FILE>
+  project        <YOUR PROJECT ID>
+  bucket         <YOUR BUCKET ID>
+  path           /path/to/the/output/file
+
+  buffer_path    <PATH OF THE BUFFER>
+</match>
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fluent-plugin-gcloud-storage. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fluent-plugin-gcloud-storage.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-gcloud-storage'
+  spec.version       = '0.1.0'
+  spec.authors       = ['Gergő Sulymosi']
+  spec.email         = ['gergo.sulymosi@gmail.com']
+
+  spec.summary       = 'Google Cloud Storage output plugin for fluentd event collector'
+  spec.description   = spec.summary
+  spec.homepage      = 'https://github.com/trekdemo/fluent-plugin-gcloud-storage'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'test-unit'
+
+  spec.add_runtime_dependency 'fluentd', '> 0.10.42'
+  spec.add_runtime_dependency 'gcloud', '~> 0.4.0'
+end

data/lib/fluent/plugin/out_gcloud_storage.rb

@@ -0,0 +1,114 @@
+module Fluent
+  class GcloudStorageOutput < TimeSlicedOutput
+    CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
+    DEFAULT_TIME_SLICE_FORMAT = '%Y%m%d'
+
+    Fluent::Plugin.register_output('gcloud_storage', self)
+
+    config_set_default :time_slice_format, DEFAULT_TIME_SLICE_FORMAT
+    config_param :project, :string, :default => nil
+    config_param :bucket,  :string, :default => nil
+    config_param :key,     :string, :default => nil
+    config_param :path,    :string, :default => nil
+    config_param :format,  :string, :default => 'out_file'
+
+    def initialize
+      super
+
+      require 'gcloud'
+    end
+
+    def configure(conf)
+      predict_time_slice_format!(conf)
+
+      super
+
+      ensure_gcloud_arguments!
+      ensure_proper_path!
+      ensure_buffer_type!
+      setup_formatter(conf)
+    end
+
+    def start
+      super
+
+      @gs_bucket = Gcloud
+        .new(@project, @key)
+        .storage
+        .bucket(@bucket)
+    end
+
+    # def shutdown; super; end
+
+    def format(tag, time, record)
+      @formatter.format(tag, time, record)
+    end
+
+    def write(chunk)
+      local_path = chunk.path
+      remote_path = generate_path(chunk)
+
+      log.debug('Writting chunk(%s) from: %s to: %s' % [chunk.key, local_path, remote_path])
+      before = Time.now.to_f
+
+      @gs_bucket.create_file(local_path, remote_path)
+
+      log.debug('Chunk(%s) upload was done in: %.2fs' % [chunk.key, (Time.now.to_f - before)])
+    end
+
+    private
+
+    def generate_path(chunk)
+      path_chunk_id = chunk_unique_id_to_str(chunk.unique_id)
+
+      Time
+        .strptime(chunk.key, @time_slice_format) # parse chunk time
+        .strftime(@path)                         # replace path placeholders
+        .gsub(CHUNK_ID_PLACE_HOLDER, path_chunk_id)
+    end
+
+    def chunk_unique_id_to_str(unique_id)
+      unique_id.unpack('C*').map{|x| x.to_s(16).rjust(2,'0')}.join('')
+    end
+
+    def predict_time_slice_format!(conf)
+      return unless (path = conf['path'])
+
+      conf['time_slice_format'] =
+        case
+        when path.index('%S') then '%Y%m%d%H%M%S'
+        when path.index('%M') then '%Y%m%d%H%M'
+        when path.index('%H') then '%Y%m%d%H'
+        else DEFAULT_TIME_SLICE_FORMAT
+        end
+    end
+
+    def ensure_gcloud_arguments!
+      fail ConfigError, "'project' must be specified." unless @project
+      fail ConfigError, "'bucket' must be specified." unless @bucket
+      fail ConfigError, "'key' must be specified." unless @key
+    end
+
+    def ensure_proper_path!
+      fail ConfigError, "'path' parameter is required" unless @path
+
+      if @path.index(CHUNK_ID_PLACE_HOLDER).nil?
+        fail Fluent::ConfigError,
+          'path must contain ${chunk_id}, which is the placeholder for buffer'\
+          'chunk.key. Google Cloud Storage does not support append operation'\
+          'on objects. Tip: Use gsutil compose command to merge objects.'
+      end
+    end
+
+    def ensure_buffer_type!
+      fail ConfigError, "'buffer_type file' is mandatory" unless @buffer_type == 'file'
+    end
+
+    def setup_formatter(conf)
+      # http://docs.fluentd.org/articles/out_file#format
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
+    end
+  end
+end
+

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-gcloud-storage
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Gergő Sulymosi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-10-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 0.10.42
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 0.10.42
+- !ruby/object:Gem::Dependency
+  name: gcloud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+description: Google Cloud Storage output plugin for fluentd event collector
+email:
+- gergo.sulymosi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/setup
+- fluent-plugin-gcloud-storage.gemspec
+- lib/fluent/plugin/out_gcloud_storage.rb
+homepage: https://github.com/trekdemo/fluent-plugin-gcloud-storage
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Google Cloud Storage output plugin for fluentd event collector
+test_files: []