fluent-plugin-presto_query 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4315ee92212f8e536132e50f6c73f62cd76be266
+  data.tar.gz: a6a0e85c5d4ac8a0823d20267ff37e290fd63877
+SHA512:
+  metadata.gz: f499a3f008b00e6d5e03b6162147f2ec038ae7d14a2508fa6b624cc90a03ea764a0e87c230b55ad9780aebb77c422bba8f27e9808a922ebc56a9d6dc7ef6ed48
+  data.tar.gz: cec161c298de23c396a28f71a8452593dbb0d7669f8e9ebad809a230f61e38167961ac6568224ff6f865d4a09e40800a13573a08603d11135213a0cf24950d9d

data/.document

@@ -0,0 +1,3 @@
+- 
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,9 @@
+Gemfile.lock
+doc/
+tmp/
+pkg/
+**/*.gem
+.yardoc/
+.bundle/
+vendor/bundle/
+.tags

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - ruby-head
+
+os:
+  - linux
+  - osx
+
+gemfile:
+  - Gemfile
+
+script: 'bundle exec rake test'

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title "fluent-plugin-presto_query Documentation" --protected

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'kramdown'
+end

data/README.md

@@ -0,0 +1,63 @@
+# fluent-plugin-presto_query [![Build Status](https://secure.travis-ci.org/toyama0919/fluent-plugin-presto_query.png?branch=master)](http://travis-ci.org/toyama0919/fluent-plugin-presto_query)
+
+Query to [Presto](https://prestodb.io/) plugin for fluentd
+
+## Examples
+```
+<source>
+  type presto_query
+  tag presto_query.exsample
+  host presto-cordinator
+  catalog store
+  schema public
+  interval 10s
+  sql select user_id, count(*) as cnt from db01.schema.conversion where user_id is not null group by user_id having count(*) > 1000
+  cron 0 * * * * # You can query at a given time, with cron syntax.
+</source>
+
+<match presto_query.exsample>
+  type stdout
+</match>
+```
+
+#### output
+```
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":1000,"cnt":3906}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":2000,"cnt":1348}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":3000,"cnt":1167}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":4000,"cnt":1503}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":5000,"cnt":1689}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":6000,"cnt":1199}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":7000,"cnt":1182}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":8000,"cnt":1079}
+2015-05-12 16:01:02 +0900 presto_query.exsample: {"user_id":9000,"cnt":2455}
+...
+```
+
+## parameter
+#TODO
+
+## Installation
+```
+fluent-gem install fluent-plugin-presto_query
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Information
+
+* [Homepage](https://github.com/toyama0919/fluent-plugin-presto_query)
+* [Issues](https://github.com/toyama0919/fluent-plugin-presto_query/issues)
+* [Documentation](http://rubydoc.info/gems/fluent-plugin-presto_query/frames)
+* [Email](mailto:toyama0919@gmail.com)
+
+## Copyright
+
+Copyright (c) 2015 Hiroshi Toyama
+

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/exsample/exsample.conf

@@ -0,0 +1,26 @@
+
+<source>
+  type presto_query
+  tag presto_query.exsample
+  host presto-cordinator
+  catalog store
+  schema public
+  sql select user_id, count(*) as cnt from pageview where user_id is not null group by user_id having count(*) > 1000
+  cron * * * * *
+</source>
+
+<source>
+  type presto_query
+  tag presto_query.exsample
+  host presto-cordinator
+  catalog store
+  schema public
+  interval 10s
+  sql select user_id, count(*) as cnt from db01.schema.conversion where user_id is not null group by user_id having count(*) > 1000
+  cron 0 * * * *
+</source>
+
+<match presto_query.exsample>
+  type stdout
+</match>
+

data/fluent-plugin-presto_query.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-presto_query"
+  gem.version       = "0.0.1"
+  gem.summary       = %q{PrestoQuery plugin for fluentd}
+  gem.description   = %q{PrestoQuery plugin for fluentd}
+  gem.license       = "MIT"
+  gem.authors       = ["Hiroshi Toyama"]
+  gem.email         = "toyama0919@gmail.com"
+  gem.homepage      = "https://github.com/toyama0919/fluent-plugin-presto_query"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "presto-client"
+  gem.add_runtime_dependency "parse-cron"
+  gem.add_development_dependency 'bundler', '~> 1.7.2'
+  gem.add_development_dependency 'fluentd', '~> 0.10.53'
+  gem.add_development_dependency 'pry', '~> 0.10.1'
+  gem.add_development_dependency 'rake', '~> 10.3.2'
+  gem.add_development_dependency 'rubocop', '~> 0.24.1'
+  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  gem.add_development_dependency 'yard', '~> 0.8'
+end

data/lib/fluent/plugin/in_presto_query.rb

@@ -0,0 +1,76 @@
+module Fluent
+  class PrestoQueryInput < Fluent::Input
+    Plugin.register_input 'presto_query', self
+
+    config_param :tag, :string
+    config_param :sql, :string
+    config_param :host, :string
+    config_param :port, :integer, default: 8080
+    config_param :user, :string, default: 'fluentd'
+    config_param :schema, :string, default: 'default'
+    config_param :catalog, :string, default: 'native'
+    config_param :interval, :time, default: '30m' # not use.
+    config_param :cron, :string, default: '0 0 * * *'
+
+    def configure(conf)
+      require 'presto-client'
+      require 'parse-cron'
+      super
+    end
+
+    def start
+      @cron_parser = CronParser.new(@cron)
+      @client = Presto::Client.new(
+        server: "#{@host}:#{@port}",
+        catalog: @catalog,
+        user: @user,
+        schema: @schema
+      )
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      Thread.kill(@thread)
+    end
+
+    def run
+      loop do
+        secs = @cron_parser.next(Time.now) - Time.now
+        log.info "next query at #{@cron_parser.next(Time.now)}. Sleep #{secs}seconds."
+        sleep secs
+        Thread.new(&method(:emit_presto_query))
+      end
+    end
+
+    def emit_presto_query
+      begin
+        log.info "sql [#{@sql}]"
+        records = exec_query(@sql)
+        records.each do |record|
+          Fluent::Engine.emit @tag, Fluent::Engine.now, record
+        end
+      rescue => e
+        log.error e
+      end
+    end
+
+    private
+    def exec_query(query)
+      return if query.nil? || query.empty?
+      @client.query(query) do |q|
+        columns = q.columns.map {|column|
+          column.name
+        }
+
+        results = []
+        q.each_row {|row|
+          results << columns.each_with_index.inject({}) {|result, (column_name, i)| 
+            result[column_name] = row[i]
+            result
+          }
+        }
+        return results
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/in_presto_query'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_presto_query.rb

@@ -0,0 +1,29 @@
+require 'helper'
+
+class PrestoQueryInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::PrestoQueryInput).configure(conf)
+  end
+
+  def test_configure_full
+    d = create_driver %q{
+      tag test
+      interval 10m
+      sql "select * from hoge"
+      host presto-cordinator
+    }
+
+    assert_equal 'test', d.instance.tag
+    assert_equal 10 * 60, d.instance.interval
+  end
+
+  def test_configure_error_when_config_is_empty
+    assert_raise(Fluent::ConfigError) do
+      create_driver ''
+    end
+  end
+end

metadata

@@ -0,0 +1,184 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-presto_query
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hiroshi Toyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: presto-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parse-cron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.53
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.53
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: PrestoQuery plugin for fluentd
+email: toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
+- Gemfile
+- README.md
+- Rakefile
+- exsample/exsample.conf
+- fluent-plugin-presto_query.gemspec
+- lib/fluent/plugin/in_presto_query.rb
+- test/helper.rb
+- test/plugin/test_in_presto_query.rb
+homepage: https://github.com/toyama0919/fluent-plugin-presto_query
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: PrestoQuery plugin for fluentd
+test_files:
+- test/helper.rb
+- test/plugin/test_in_presto_query.rb
+has_rdoc: