fluent-plugin-deskcom 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 952794ba64a64955c3fbd9f599c5faedc81c3374
-  data.tar.gz: 8619e94793db9f42d657b19e0f30207233a7e8bc
+  metadata.gz: f8e389008903fec965494fca734fed91ab1bb777
+  data.tar.gz: 534bf1b6679a6741e9bf9fbfe628f94d0a079053
 SHA512:
-  metadata.gz: b2e0bc35b53bfdb8fd92fee8ffe933ded9d45c3d2d276d6202542ac0bacf5e4dfcac526df97ca32f805806df8037db295ff4d5264cb0edd6b43a0fb5bc9bcd29
-  data.tar.gz: 55e0f6aba3af35167ff91d506b0faa0a516cf3b18cc7908d4beb2779da639327f84267557dec8982310d32c9a1257763491aae21ab339242457113cd9714b1d0
+  metadata.gz: ef1ef41f236a754ee61035bf4e80864e2bf77c06ae982b5092d40b17daa3bce0b261ed976097c7b7331402cc6f865a7481600b4c4540894b4ea0c6ee7b6766fe
+  data.tar.gz: 297c61cec432104e6142b4e0e13c747717f4dac00c29f4e1141bdac7d2a106b84c7d0f7cf640804db568fbc06f2a656c9346d2ae51e0dc20fba4cf4f37b5d42f

data/fluent-plugin-deskcom.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-deskcom"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["Toru Takahashi"]
   spec.email         = ["torutakahashi.ayashi@gmail.com"]
   spec.summary       = %q{Input plugin to collect data from Deskcom.}

data/lib/fluent/plugin/in_deskcom.rb

@@ -2,12 +2,14 @@ module Fluent
 class DeskcomInput < Fluent::Input
   Fluent::Plugin.register_input('deskcom', self)
 
-  # un-support yet: nest flat
+  # unsupported yet: nest flat
   OUTPUT_FORMAT_TYPE = %w(simple)
-  # un-support yet: brand article reply ~
+  # unsupported yet: brand article reply ~
   INPUT_API_TYPE = %w(cases replies)
   DEFAULT_PER_PAGE = 50
 
+  SORT_DIRECTION_TYPE=%w(asc desc)
+
   config_param :subdomain,            :string, :default => nil
   config_param :consumer_key,         :string, :default => nil
   config_param :consumer_secret,      :string, :default => nil
@@ -19,6 +21,7 @@ class DeskcomInput < Fluent::Input
   config_param :tag,                  :string, :default => nil
   config_param :time_column,          :string, :default => nil
   config_param :interval,             :integer,:default => 5
+  config_param :sort_direction,       :string, :default => 'asc'
 
   def initialize
     super
@@ -38,6 +41,10 @@ class DeskcomInput < Fluent::Input
       raise Fluent::ConfigError, "input_api value undefined #{@input_api}"
     end
 
+    if !SORT_DIRECTION_TYPE.include?(@sort_direction)
+      raise Fluent::ConfigError, "sort_direction value undefined #{@sort_direction}"
+    end
+
     if !@consumer_key || !@consumer_secret || !@oauth_token || !@oauth_token_secret
       raise Fluent::ConfigError, "missing values in consumer_key or consumer_secret or oauth_token or oauth_token_secret"
     end
@@ -48,7 +55,6 @@ class DeskcomInput < Fluent::Input
 
     @tick = @interval * 60
 
-    @stored_time = load_store_file
     @per_page = DEFAULT_PER_PAGE
 
     Desk.configure do |config|
@@ -74,47 +80,61 @@ class DeskcomInput < Fluent::Input
   def run
     while true
       @started_time = Time.now.to_i
+      @stored_time = load_store_file(@store_file)
       get_stream
-      save_store_file unless !@store_file
+      save_store_file(@store_file, @started_time) unless !@store_file
       sleep @tick
     end
   end
 
-  def get_stream()
-    page = 0
-    if @input_api == 'cases' then
+  def get_stream
+    page = 1
+    loop do
+      cases = nil
       begin
-        page = page + 1
-        cases = Desk.cases(:since_updated_at => @stored_time, :page => page, :per_page => @per_page)
-        # Sleep for rate limit
-        # ToDo: Check body "Too Many Requests" and Sleep
-        sleep(2)
+        cases = Desk.cases(:since_updated_at => @stored_time, :max_updated_at => @started_time,
+                           :page => page, :per_page => @per_page,
+                           :sort_field => 'updated_at', :sort_direction => @sort_direction)
+      rescue Desk::NotFound => e
+        $log.info "No more records: #{e.message}"
+        break
+      end
 
+      if @input_api == 'cases' then
         cases.each do |c|
           get_content(c)
         end
-        $log.info "Case total entry: #{cases.total_entries} page: #{page}"
-      end while (cases.total_entries/@per_page.to_f).ceil > page
-    elsif @input_api == 'replies'
-      begin
-        page = page + 1
-        cases = Desk.cases(:since_updated_at => @stored_time, :page => page, :per_page => @per_page)        
-        # Sleep for rate limit
-        # ToDo: Check body "Too Many Requests" and Sleep
-        sleep(2)
+        $log.info "Case total entries: #{cases.total_entries} page: #{page}"
 
+      elsif @input_api == 'replies'
         cases.each do |c|
           Desk.case_replies(c.id).each do |r|
-            # Sleep for rate limit
-            # ToDo: Check body "Too Many Requests" and Sleep
-            sleep(2)
-            
             r[:case_id] = c.id
             get_content(r) if c.count > 0
           end
         end
-        $log.info "Case total entry include reply #{cases.total_entries} page: #{page}"
-      end while (cases.total_entries/@per_page.to_f).ceil > page
+        $log.info "Case total entries with replies: #{cases.total_entries} page: #{page}"
+      end
+
+      page = page + 1
+      # if getting above 500 pages limit for a search
+      #   (http://dev.desk.com/API/cases/#list), reset the reference point and
+      #   the page count to 1
+      if page >= 500
+        require 'time'
+        if @sort_direction == 'asc'
+          # if sorting by ascending 'updated_at', reset the lower filter limit
+          #   to focus on the upper part of the records that would reside in the
+          #   'pages' past 500
+          @stored_time = Time.parse(cases.map(&:updated_at).sort.last).to_i
+        else
+          # if sorting by descending 'updated_at', reset the upper filter limit
+          #   to focus on the lower part of the records that would reside in the
+          #   'pages' past 500
+          @started_time = Time.parse(cases.map(&:updated_at).sort.first).to_i
+        end
+        page = 1
+      end
     end
   rescue => e
     $log.error "deskcom run: #{e.message}"
@@ -125,14 +145,6 @@ class DeskcomInput < Fluent::Input
     when 'simple'
       record = Hash.new
       status.each_pair do |k,v|
-        # @stored_time <= store data's updated time < @started_time
-        if (k == 'updated_at') then
-          at_time = Time.parse(v).to_i
-          if (at_time >= @started_time) || (at_time < @stored_time) then
-            next
-          end
-        end
-
         if (!@time_column.nil? && k == "#{@time_column}") then
           @time_value = Time.parse(v).to_i rescue nil
         end
@@ -158,34 +170,29 @@ class DeskcomInput < Fluent::Input
     $log.error "deskcom get_content: #{e.message}"
   end
 
-  # => int
-  def load_store_file
-    begin
-      f = Pathname.new(@store_file)
-      stored_time = 0
-      f.open('r') do |f|
-        stored = YAML.load_file(f)
-        stored_time = stored[:time].to_i
-      end
-      $log.info "deskcom: Load #{@store_file}: #{@stored_time}"
-    rescue => e
-      $log.warn "deskcom: Can't load store_file #{e.message}"
-      return 0
+  def load_store_file(store_file)
+    f = Pathname.new(store_file)
+    stored_time = 0
+    f.open('r') do |f|
+      stored = YAML.load_file(f)
+      stored_time = stored[:time].to_i
     end
+    $log.info "deskcom: Load #{store_file}: #{stored_time}"
     return stored_time
+  rescue => e
+    $log.warn "deskcom: Can't load store_file #{e.message}"
+    return 0
   end
 
-  def save_store_file
-    begin
-      f = Pathname.new(@store_file)
-      f.open('w') do |f|
-        data = {:time => @started_time}
-        YAML.dump(data, f)
-      end
-      $log.info "deskcom: Save started_time: #{@started_time} to #{@store_file}"
-    rescue => e
-      $log.warn "deskcom: Can't save store_file #{e.message}"
+  def save_store_file(store_file, time)
+    f = Pathname.new(store_file)
+    f.open('w') do |f|
+      data = {:time => time}
+      YAML.dump(data, f)
     end
+    $log.info "deskcom: Save started_time: #{time} to #{store_file}"
+  rescue => e
+    $log.warn "deskcom: Can't save store_file #{e.message}"
   end
 
 end

metadata

@@ -1,69 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-deskcom
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Toru Takahashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-19 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: desk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: fluent Input plugin to collect data from Deskcom.
@@ -73,7 +73,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -92,17 +92,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Input plugin to collect data from Deskcom.