fluent-plugin-json-api 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d1006b0671e52a065430a413925fb13586297a3f
+  data.tar.gz: 419f04fa670350c08beb147d11fb8b048af6311f
+SHA512:
+  metadata.gz: dba458a5d507da398eb37ced674b2bea989216cdeea110e43482a8b29952c4f932f48eb02ec9fe11a6db09f91c364c2b2fdf88e7cd1c7bd3848f327ad2428099
+  data.tar.gz: c679b8fd5b329027fcb20b258ca021d81499e50728034ad1661562d6015830cfaf83827d2e9815b59ae8faf7c7b1b5797b579b7c6f98aa0dbe19aa616e24a671

data/.gitignore

@@ -0,0 +1,25 @@
+# For TextMate, emacs, vim
+*.gem
+*.rbc
+*.swp
+*.tmproj
+*~
+.\#*
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+\#*
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+pkg/*
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+tmtags

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-json-api.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Freddie FUJIWARA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Fluent::Plugin::Json::Api
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-json-api'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-json-api
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-json-api.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-json-api"
+  s.version     = "0.0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Freddie Fujiwara"]
+  s.date        = %q{2013-09-29}
+  s.email       = "github@ze.gs"
+  s.homepage    = "http://github.com/freddiefujiwara/fluent-plugin-json-api"
+  s.summary     = "json-api plugin for Fluentd"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency %q<fluentd>, ["~> 0.10.0"]
+end

data/lib/fluent/plugin/in_json_api.rb

@@ -0,0 +1,77 @@
+module Fluent
+    class JsonApiError < StandardError
+    end
+    class JsonApiInput < Input
+        Plugin.register_input('json_api', self)
+
+        config_param :url,      :string,  :default => 'http://pipes.yahoo.com/pipes/pipe.run?_id=c9b9df32b4c3e0ccbe4547ae7e00ed2f&_render=json&condition=d7D&genre=100533&page=__PAGE__'
+        config_param :rules   , :string
+        config_param :sleep   , :integer, :default =>  0
+        config_param :tag,      :string
+
+        attr_reader :urls
+        def initialize
+            super
+            require 'json'
+            require 'net/http'
+            @urls = []
+        end
+
+        def configure(config)
+            super
+
+            begin
+                result = []
+                (eval config['rules']).each_pair do |key,replecements|
+                    combinations = []
+                    (replecements.to_a).each do |replacement|
+                        combinations << {key => replacement}
+                    end 
+                    result << combinations
+                end
+
+                @rules = result[0].product(*result[1..-1])
+            rescue SyntaxError,StandardError => e 
+                raise Fluent::ConfigError.new "rules has some errors #{e}"
+            end
+
+            raise Fluent::ConfigError unless ['http','https'].include? URI.parse(config['url']).scheme
+            @rules.each do |pair|
+                url = config['url']
+                pair.each do |key_replacements|
+                    key_replacements.each do |key,replacement|
+                        url = url.gsub(key.to_s,replacement.to_s)
+                    end
+                end
+                @urls.push url
+            end
+        end
+
+        def start
+            super
+            @thread = Thread.new(&method(:run))
+        end
+
+        def run
+            loop {
+                @urls.each do |url|
+                    Engine.emit @tag, Engine.now , crawl(url)
+                    sleep @sleep if @sleep.to_i > 0
+                end
+            }
+        end
+
+        def crawl(url)
+            response = Net::HTTP.get_response(URI.parse(url))
+            case response
+            when Net::HTTPSuccess
+                return JSON.parse response.body
+            end
+            raise Fluent::JsonApiError.new
+        end
+
+        def shutdown
+            Thread.kill(@thread)
+        end
+    end
+end

data/test/plugin/in_json_api.rb

@@ -0,0 +1,110 @@
+require 'fluent/test'
+require 'json'
+require 'fluent/plugin/in_json_api'
+
+class FileInputTest < Test::Unit::TestCase
+    def setup
+        Fluent::Test.setup
+
+        @d = create_driver %[
+            url   http://pipes.yahoo.com/pipes/pipe.run?_id=c9b9df32b4c3e0ccbe4547ae7e00ed2f&_render=json&condition=d7D&genre=__GENRE__&page=__PAGE__
+            rules   {'__PAGE__' => (1..10),'__GENRE__' => [1,100533]} 
+            sleep       1
+            tag     input.json
+        ]
+        @time = Time.now.to_i
+    end
+
+    def create_driver(config = CONFIG)
+        Fluent::Test::OutputTestDriver.new(Fluent::JsonApiInput).configure(config)
+    end
+
+    def test_configure
+        assert_equal 'http://pipes.yahoo.com/pipes/pipe.run?_id=c9b9df32b4c3e0ccbe4547ae7e00ed2f&_render=json&condition=d7D&genre=__GENRE__&page=__PAGE__'   , @d.instance.url
+        assert_equal @d.instance.rules.size, 20 #10(page) x 2(genre)
+        assert_equal 1           , @d.instance.sleep
+        assert_equal 'input.json', @d.instance.tag
+        [ %[
+                url   hoge
+                rules   {'__PAGE__' => (1..10)}
+                sleep       1
+                tag     input.json
+            ], #url wrong
+          %[
+                url   ftp://hoge.com
+                rules   {'__PAGE__' => (1..10)}
+                sleep       1
+                tag     input.json
+            ], #url must start at http
+          %[
+                url   http://hoge.com
+                sleep       1
+                rules   {'__PAGE__' => (1..10)}
+            ], #no tag
+          %[
+                url   http://hoge.com
+                sleep       1
+                rules   {__PAGE__}
+                tag     input.json
+            ], #rules must be {'key' => array}
+          %[
+                url   http://hoge.com
+                sleep       1
+                rules   (1..2)
+                tag     input.json
+            ], #rules must be {'key' => array}
+        ].each do |config|
+            assert_raise Fluent::ConfigError do
+                create_driver config
+            end
+        end
+    end
+
+    def test_urls
+        assert_equal @d.instance.urls.size, 20
+        index = 0
+        (1..10).each do |page|
+            [1,100533].each do |genre|
+                assert_equal "http://pipes.yahoo.com/pipes/pipe.run?_id=c9b9df32b4c3e0ccbe4547ae7e00ed2f&_render=json&condition=d7D&genre=#{genre}&page=#{page}", @d.instance.urls[index]
+
+                index += 1
+            end
+        end
+    end
+
+    def test_crawls
+        for page in 1..1
+            url = "http://pipes.yahoo.com/pipes/pipe.run?_id=c9b9df32b4c3e0ccbe4547ae7e00ed2f&_render=json&condition=d7D&genre=100533&page=#{page}"
+            response = @d.instance.crawl url
+            assert_not_nil response
+            assert_equal response['count'], 30
+            assert_equal response['value']['items'].size, 30
+            response['value']['items'].each do |item|
+                #content
+                assert item['author']['content'].to_f >= 0.0
+                assert item['author']['content'].to_f <= 5.0
+
+                #link
+                assert_equal URI.parse(item['link']).scheme, 'http' if item['link']
+                #title
+                assert_equal URI.parse(item['title']).scheme, 'http' if item['title']
+
+                #description
+                assert_not_nil item['description']
+
+                #pubDate
+                assert_not_nil item['pubDate']['content']
+            end
+        end
+
+        #parse error
+        assert_raise JSON::ParserError do
+            @d.instance.crawl 'http://www.rakuten.co.jp'
+        end
+
+        #404 Not found
+        assert_raise Fluent::JsonApiError do
+            @d.instance.crawl 'http://www.google.com/404'
+        end
+    end
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-json-api
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Freddie Fujiwara
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+description: 
+email: github@ze.gs
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-json-api.gemspec
+- lib/fluent/plugin/in_json_api.rb
+- test/plugin/in_json_api.rb
+homepage: http://github.com/freddiefujiwara/fluent-plugin-json-api
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0.preview2
+signing_key: 
+specification_version: 4
+summary: json-api plugin for Fluentd
+test_files: []