fluent-plugin-rss 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 453767bbb3e9f90ec40e64e97a9b1b3f725a60b1
+  data.tar.gz: 508affd5097e64099f17e6cc64afce53947648a2
+SHA512:
+  metadata.gz: b0d42924eb14029622f283948a096b31feed29e20a4b6ea98997b9eb8352a9abf95c0f4f927ab5791d5967f6b26d67078f14fe31d33ad950473f1bec23d96dcb
+  data.tar.gz: 2927c5eecefaee685e84a392fea38cacd6814b879e9c6363ce243d4b56239a9ff136d0a29f5ad294b4cd982282c6f621c9e85b5cca7d37cbfea8163fc3373f26

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-rss.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-rss (0.0.1)
+      fluentd
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.2.1)
+    fluentd (0.10.44)
+      cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
+      http_parser.rb (~> 0.5.1)
+      json (>= 1.4.3)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      sigdump (~> 0.2.2)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.5.3)
+    json (1.8.1)
+    msgpack (0.5.8)
+    rake (10.2.0)
+    sigdump (0.2.2)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.5)
+  fluent-plugin-rss!
+  rake

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Shunsuke Michii
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# fluent-plugin-rss
+
+fetch RSS feed items and emit to fluentd stream.
+
+## Usage
+
+```
+<source>
+  tag rss.nytimes                                     # Required: tag to emit
+  url http://example.com/input-your-rss-feed-here.rdf # Required: feed url
+  interval 10m                                        # Optional: fetch interval (default: 5 min.)
+  attrs date, title                                   # Optional: attributes to emit stream
+</source>
+```
+
+## Todo
+
+- more tests
+- more documentation (if you needed)
+
+* * *
+
+## Copyright
+
+Copyright &copy; 2014 Shunsuke Michii.
+All rights reserved.
+
+This software is released under MIT License (see LICENCE file).
+
+## Contributing
+
+Patches Welcome!
+
+1. Fork it ( http://github.com/<my-github-username>/fluent-plugin-rss/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-rss.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-rss"
+  spec.version       = "0.0.1"
+  spec.authors       = ["harukasan"]
+  spec.email         = ["harukasan@pixiv.com"]
+  spec.summary       = %q{Fluent input plugin to fetch RSS feed items}
+  spec.description   = %q{Fluent input plugin to fetch RSS feed items.}
+  spec.homepage      = "https://github.com/harukasan/fluent-plugin-rss"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^test/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_rss.rb

@@ -0,0 +1,55 @@
+require 'rss'
+
+module Fluent
+  class RSSInput < Fluent::Input
+    Plugin.register_input 'rss', self
+
+    config_param :tag, :string
+    config_param :url, :string
+    config_param :interval, :time, default: '5m'
+    config_param :attrs, :string, default: 'date, title, description'
+
+    def configure(conf)
+      super
+
+      @attrs = @attrs.split(',').map {|attr| attr.strip }
+      @current_time = Time.now
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      Thread.kill(@thread)
+    end
+
+    def run
+      loop do
+        Thread.new(&method(:emit_rss))
+        sleep @interval
+      end
+    end
+
+    def emit_rss
+      begin
+        next_current_time = @current_time
+        rss = RSS::Parser.parse(@url)
+        rss.items.each do |item|
+          record = {}
+          @attrs.each do |attr|
+            record[attr] = item.send(attr) if item.send(attr)
+          end
+          time = Time.parse item.date.to_s
+          if time > @current_time
+            Fluent::Engine.emit @tag, Time.parse(item.date.to_s), record
+            next_current_time = time if time > next_current_time
+          end
+        end
+        @current_time = next_current_time
+      rescue => e
+        log.error e
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/in_rss'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_rss.rb

@@ -0,0 +1,32 @@
+require 'helper'
+
+class RSSInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::RSSInput).configure(conf)
+  end
+
+  def test_configure_full
+    d = create_driver %q{
+      tag test
+      url http://example.com/example.rss
+      interval 10m
+      attrs date, title
+    }
+
+    assert_equal 'test', d.instance.tag
+    assert_equal 'http://example.com/example.rss', d.instance.url
+    assert_equal 10 * 60, d.instance.interval
+    assert_equal ['date', 'title'], d.instance.attrs
+  end
+
+  def test_configure_error_when_config_is_empty
+    assert_raise(Fluent::ConfigError) do
+      create_driver ''
+    end
+  end
+end
+

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-rss
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- harukasan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent input plugin to fetch RSS feed items.
+email:
+- harukasan@pixiv.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-rss.gemspec
+- lib/fluent/plugin/in_rss.rb
+- test/helper.rb
+- test/plugin/test_in_rss.rb
+homepage: https://github.com/harukasan/fluent-plugin-rss
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Fluent input plugin to fetch RSS feed items
+test_files:
+- test/helper.rb
+- test/plugin/test_in_rss.rb