fluent-plugin-deskcom 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5641dbda7a204f52d882965061aea9a383f83730
-  data.tar.gz: 9170d67ca166201d7bc734efec235d927c92a017
+  metadata.gz: 952794ba64a64955c3fbd9f599c5faedc81c3374
+  data.tar.gz: 8619e94793db9f42d657b19e0f30207233a7e8bc
 SHA512:
-  metadata.gz: 9057d3893407ec129cc8cf68f6280fb329ce1578c51628085ac5c19bc61a5f2dc970c28ec47d6414fe0ca05529522fe9894e3fee4f24ce2c3363f03a0384f0aa
-  data.tar.gz: 3ccee42a011fa0a5c40fb3c3890c8808b0bcde0a52b971764746303757749750bccb5f061017660763553a1fccf06adc706341ab5504d50e742c17e1b70777db
+  metadata.gz: b2e0bc35b53bfdb8fd92fee8ffe933ded9d45c3d2d276d6202542ac0bacf5e4dfcac526df97ca32f805806df8037db295ff4d5264cb0edd6b43a0fb5bc9bcd29
+  data.tar.gz: 55e0f6aba3af35167ff91d506b0faa0a516cf3b18cc7908d4beb2779da639327f84267557dec8982310d32c9a1257763491aae21ab339242457113cd9714b1d0

data/README.md

@@ -7,6 +7,12 @@ This plugin is still experimental.
 This plugin depends on [desk library](https://github.com/chriswarren/desk)
 Also, please see [Desk API v2](http://dev.desk.com/)
 
+## Quick Start
+
+If you want to use fluent-plugin-deskcom on Heroku, you can start easily.
+Please see the following page, and click Deploy to Heroku button!
+https://github.com/toru-takahashi/heroku-td-agent
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -23,8 +29,26 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-deskcom
 
+### Get Desk.com API key
+
+Your Desk.com API key, secret, token and token secret can be found by following this guide:
+
+1. Log in to your Desk.com admin panel using your personal URL (http://your-account.desk.com/admin).
+2. Click on Settings:
+![step2](http://gyazo.com/020298636ef217e0d3bb7ee7b6a0bd31.png)
+3. From the left menu select API > My Applications.
+4. If you don’t already have an application defined for TreasureData, create one by clicking on the “Add API Application” button and enter the following information:
+![step4-1](http://gyazo.com/803b486ed3a86c77af55819e83d4d85b.png)
+![step4-2](http://gyazo.com/97a5624030f1bc05fe040e386d30f7dd.png)
+5. Find your Key and Secret codes for the TreasureData API application:
+![step5](http://gyazo.com/895bf963f01b65254e38c534855a1914.png)
+6. Find your Token and Token secret by clicking on the Your Access Token link:
+![step6](http://gyazo.com/7f057fb4b6b88b616f9895d4d5b69e9d.png)
+
 ## Usage
 
+The followings are examples of fluentd.conf(td-agent.conf)
+
 ### Cases Config Sample
 
 ```ruby
@@ -151,8 +175,6 @@ Or install it yourself as:
 </match>
 ```
 
-Please check store_file's permission to write by fluentd.
-
 ## TO DO
 
 - Allows over 2 input_api (Ex. input_api cases,replies)

data/fluent-plugin-deskcom.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-deskcom"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["Toru Takahashi"]
   spec.email         = ["torutakahashi.ayashi@gmail.com"]
   spec.summary       = %q{Input plugin to collect data from Deskcom.}

data/lib/fluent/plugin/in_deskcom.rb

@@ -18,6 +18,7 @@ class DeskcomInput < Fluent::Input
   config_param :input_api,            :string, :default => 'cases'
   config_param :tag,                  :string, :default => nil
   config_param :time_column,          :string, :default => nil
+  config_param :interval,             :integer,:default => 5
 
   def initialize
     super
@@ -45,9 +46,9 @@ class DeskcomInput < Fluent::Input
       $log.warn("stored_time_file path is missing")
     end
 
+    @tick = @interval * 60
 
     @stored_time = load_store_file
-    @started_time = Time.now.to_i
     @per_page = DEFAULT_PER_PAGE
 
     Desk.configure do |config|
@@ -65,32 +66,56 @@ class DeskcomInput < Fluent::Input
   end
 
   def shutdown
-    @thread.kill
+    super
+    @thread.terminate
+    @thread.join
   end
 
   def run
-    page = 1
+    while true
+      @started_time = Time.now.to_i
+      get_stream
+      save_store_file unless !@store_file
+      sleep @tick
+    end
+  end
+
+  def get_stream()
+    page = 0
     if @input_api == 'cases' then
       begin
+        page = page + 1
         cases = Desk.cases(:since_updated_at => @stored_time, :page => page, :per_page => @per_page)
+        # Sleep for rate limit
+        # ToDo: Check body "Too Many Requests" and Sleep
+        sleep(2)
+
         cases.each do |c|
           get_content(c)
         end
-        page = page + 1
-      end while cases.total_entries > (@per_page*page)
+        $log.info "Case total entry: #{cases.total_entries} page: #{page}"
+      end while (cases.total_entries/@per_page.to_f).ceil > page
     elsif @input_api == 'replies'
       begin
-        cases = Desk.cases(:since_updated_at => @stored_time, :page => page, :per_page => @per_page)
+        page = page + 1
+        cases = Desk.cases(:since_updated_at => @stored_time, :page => page, :per_page => @per_page)        
+        # Sleep for rate limit
+        # ToDo: Check body "Too Many Requests" and Sleep
+        sleep(2)
+
         cases.each do |c|
           Desk.case_replies(c.id).each do |r|
+            # Sleep for rate limit
+            # ToDo: Check body "Too Many Requests" and Sleep
+            sleep(2)
+            
             r[:case_id] = c.id
             get_content(r) if c.count > 0
           end
         end
-        page = page + 1
-      end while cases.total_entries > (@per_page*page)
+        $log.info "Case total entry include reply #{cases.total_entries} page: #{page}"
+      end while (cases.total_entries/@per_page.to_f).ceil > page
     end
-    save_store_file unless !@store_file
   rescue => e
     $log.error "deskcom run: #{e.message}"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-deskcom
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Toru Takahashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-02 00:00:00.000000000 Z
+date: 2015-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler