fluent-plugin-feedly 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+/.bundle/
+/Gemfile.lock
+/pkg/
+/tmp/
+/vendor/

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-feedly.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- Kentaro Yoshida
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,135 @@
+# fluent-plugin-feedly
+
+## Overview
+
+Fluentd input plugin to fetch RSS/ATOM feed via Feedly Could.
+
+## Dependencies
+
+* Ruby 1.9.3+
+* Fluentd 0.10.54+
+
+## Installation
+
+install with gem or fluent-gem command as:
+
+`````
+# for system installed fluentd
+$ gem install fluent-plugin-feedly
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-feedly
+`````
+
+## Configuration
+
+`````
+<source>
+  type             feedly
+
+  # Set feedly access token
+  # ref. https://feedly.com/v3/auth/dev
+  access_token     ACCESS_TOKEN    # Required
+
+  # Set file-path to store last fetched article position
+  state_file       /var/log/td-agent/feedly.state  # Required
+
+  # Set output tag
+  tag              input.feedly    # Required
+  
+  # List subscribe categories in your feedly account with JSON Array
+  subscribe_categories  ["global.all"]  # Optional (default: global.all)
+
+  # Set update checking frequency
+  run_interval     30m             # Optional (default: 10m)
+
+  # Set bulk read size
+  fetch_count      20              # Optional (default: 20)
+  
+  # fetching range of time within 30d
+  fetch_time_range 3d              # Optional (default: 3d)
+  
+  # fetching range of time for initial startup within 30d
+  fetch_time_range_on_startup 2w   # Optional (default: 2w)
+
+  # Using sandbox account
+  enable_sandbox   false           # Optional (default: false)
+  
+  # Set log level for this plugin. To see debug level, set 'debug' for this value.
+  # it can see at stdout as like `$ tail -f /var/log/td-agent/td-agent.log`
+  log_level        info 　　　　　　 # Optional (default: info)
+</source>
+`````
+
+**note** : The `subscribe_categories` is also supported with single or multi line configuration like below.
+
+```
+# single line
+subscribe_categories ["先端技術", "mysql"]
+
+# multi line
+subscribe_categories [
+  "先端技術",
+  "mysql"
+]
+```
+
+## Usage
+
+After installed this plugin, executing fluentd with following configuration.
+
+```
+$ cat /etc/td-agent/td-agent.conf
+<source>
+  type             feedly
+  access_token     YOUR_ACCESS_TOKEN
+  state_file       /var/log/td-agent/feedly.state
+  tag              input.feedly
+  run_interval     30m
+  fetch_time_range 1h
+  fetch_time_range_on_startup 3h
+  log_level        debug
+</source>
+
+<match input.feedly>
+  type             file
+  path             /tmp/feedly*.json
+  symlink_path     /tmp/feedly.json
+  format           json
+  append           true
+</match>
+```
+
+You can see the behavior about this plugin with this command.
+
+```
+# to check stdout of this plugin
+$ tail -f /var/log/td-agent/td-agent.log
+2014-10-16 14:47:01 +0900 [debug]: Feedly: fetched articles. articles=416 request_option={:count=>1000, :continuation=>"148cfb7f516:9371a3c:726280cf", :newerThan=>1412228787000}
+2014-10-16 15:02:02 +0900 [debug]: Feedly: fetched articles. articles=492 request_option={:count=>1000, :continuation=>nil, :newerThan=>1413428521000}
+```
+
+```
+# to check fetched articles
+$ tail -f /tmp/feedly.json | jq "."
+```
+
+## TODO
+
+Pull requests are very welcome!!
+
+## Copyright
+
+Copyright © 2014- Kentaro Yoshida ([@yoshi_ken](https://twitter.com/yoshi_ken))
+
+## License
+
+Apache License, Version 2.0
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-feedly/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/fluent-plugin-feedly.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-feedly"
+  #spec.version       = Fluent::Plugin::Feedly::VERSION
+  spec.version       = "0.0.1"
+  spec.authors       = ["Kentaro Yoshida"]
+  spec.email         = ["y.ken.studio@gmail.com"]
+  spec.summary       = %q{Fluentd input plugin to fetch RSS/ATOM feed via Feedly Could.}
+  #spec.description   = %q{TODO: Write a longer description. Optional.}
+  spec.homepage      = "https://github.com/y-ken/fluent-plugin-feedly"
+  spec.license       = "Apache 2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd"#, "~> 0.10.54"
+  spec.add_runtime_dependency "feedlr"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fluent/plugin/in_feedly.rb

@@ -0,0 +1,148 @@
+# -*- encoding: utf-8 -*-
+
+module Fluent
+  class FeedlyInput < Fluent::Input
+    Plugin.register_input('feedly', self)
+
+    config_param :access_token, :string
+    config_param :state_file, :string
+    config_param :tag, :string
+
+    config_param :subscribe_categories, :array, :default => ['global.all']
+    config_param :run_interval, :time, :default => 60*10 #10m
+    config_param :fetch_count, :integer, :default => 20
+    config_param :fetch_time_range, :time, :default => 60*60*24*3 #3d
+    config_param :fetch_time_range_on_startup, :time, :default => 60*60*24*14 #2w
+    config_param :enable_sandbox, :bool, :default => false
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    def initialize
+      require 'feedlr'
+      require 'digest/sha2'
+
+      super
+    end
+
+    def configure(conf)
+      super
+
+      if not @fetch_count >= 20 && @fetch_count <= 10000
+        raise Fluent::ConfigError, "Feedly: fetch_count param (#{@fetch_count}) should be between 20 and 10000."
+      end
+
+      @client = Feedlr::Client.new(
+        oauth_access_token: @access_token,
+        sandbox: @enable_sandbox,
+      )
+    end
+
+    def start
+      @profile_id = @client.user_profile.id
+      @state_store = StateStore.new(@state_file)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      Thread.kill(@thread)
+    end
+
+    def run
+      @initial_loop = true
+      loop do
+        begin
+          fetch
+        rescue => e
+          log.error "Feedly: unexpected error has occoured.", :error => e.message, :error_class => e.class
+          log.error_backtrace e.backtrace
+          sleep @run_interval
+          retry
+        end
+        sleep @run_interval
+      end
+    end
+
+    def fetch
+      @subscribe_categories.each do |category_name|
+        category_id = "user/#{@profile_id}/category/#{category_name}"
+        fetch_time_range = get_fetch_time_range
+        loop {
+          request_option = { count: @fetch_count, continuation: get_continuation_id, newerThan: fetch_time_range }
+          cursor = @client.stream_entries_contents(category_id, request_option)
+          cursor.items.each do |item|
+            Engine.emit(@tag, Engine.now, item)
+          end
+          log.debug "Feedly: fetched articles.", articles: cursor.items.size, request_option: request_option
+          set_continuation_id(cursor.continuation)
+          break if get_continuation_id.nil?
+        }
+      end
+    end
+
+    def get_fetch_time_range
+      if @initial_loop
+        @initial_loop = false
+        range = @fetch_time_range_on_startup
+      else
+        range = @fetch_time_range
+      end
+      return (Time.now.to_i - range ) * 1000
+    end
+
+    def subscribe_categories_hash
+      Digest::SHA512.digest(@subscribe_categories.sort.join(''))
+    end
+
+    def set_continuation_id(continuation_id)
+      @state_store.set("continuation", {
+        id: continuation_id,
+        subscribe_categories_hash: subscribe_categories_hash
+      })
+      @state_store.update!
+    end
+
+    def get_continuation_id
+      record = @state_store.get('continuation')
+      if subscribe_categories_hash == record[:subscribe_categories_hash]
+        return record[:id]
+      else
+        return nil
+      end
+    end
+
+    # implementation has copied from its code.
+    # https://github.com/fluent/fluent-plugin-sql/blob/master/lib/fluent/plugin/in_sql.rb
+    class StateStore
+      def initialize(path)
+        @path = path
+        if File.exists?(@path)
+          @data = YAML.load_file(@path)
+          if @data == false || @data == []
+            # this happens if an users created an empty file accidentally
+            @data = {}
+          elsif !@data.is_a?(Hash)
+            raise "state_file on #{@path.inspect} is invalid"
+          end
+        else
+          @data = {}
+        end
+      end
+
+      def set(key, data)
+        @data.store(key.to_sym, data)
+      end
+
+      def get(key)
+        @data[key.to_sym] ||= {}
+      end
+
+      def update!
+        File.open(@path, 'w') {|f|
+          f.write YAML.dump(@data)
+        }
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_feedly'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_mysql.rb

@@ -0,0 +1,30 @@
+require 'helper'
+
+class FeedlyInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    access_token     YOUR_ACCESS_TOKEN
+    state_file       /var/log/td-agent/feedly.state
+    tag              input.feedly
+    run_interval     30m
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::FeedlyInput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    d = create_driver(CONFIG)
+    assert_equal 'YOUR_ACCESS_TOKEN', d.instance.access_token
+    assert_equal '/var/log/td-agent/feedly.state', d.instance.state_file
+    assert_equal 1800, d.instance.run_interval
+    assert_equal 'input.feedly', d.instance.tag
+  end
+end
+

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-feedly
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kentaro Yoshida
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: feedlr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- y.ken.studio@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-feedly.gemspec
+- lib/fluent/plugin/in_feedly.rb
+- test/helper.rb
+- test/plugin/test_in_mysql.rb
+homepage: https://github.com/y-ken/fluent-plugin-feedly
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd input plugin to fetch RSS/ATOM feed via Feedly Could.
+test_files:
+- test/helper.rb
+- test/plugin/test_in_mysql.rb