embulk-plugin-input-sfdc-event-log-files 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2819d1e1af9032508bf81c7485f1faa33aecf945
+  data.tar.gz: d4783eecda6aa5761761874dbc997cecb34f7c9c
+SHA512:
+  metadata.gz: 72330bfce5fb1d060df9b15a2684aff8d2a56cfe3f3ae53d5df26b6ebe5e2b62420f5b39873e8ae8a9813b42d7b21ed1932ec982e453bfd2e0981d403cc38be5
+  data.tar.gz: 754c34759df13e87dbeabd8b52dc512b8d2fbad13eb704a1a590bdd3a55b8ae12c781e18262db1bf8e78592a1a8d6753ab75000c1560702ef629a159c4f7bea2

data/README.md

@@ -0,0 +1,6 @@
+embulk-plugin-input-sfdc-event-log-files
+========================================
+
+Embulk plugin for Salesforce.com Event Log Files input
+
+To be written...

data/lib/embulk/input_sfdc_event_log_files.rb

@@ -0,0 +1,115 @@
+require 'httpclient'
+require 'json'
+require 'time'
+require 'csv'
+
+module Embulk
+
+  class InputSfdcEventLogFiles < InputPlugin
+    Plugin.register_input('sfdc_event_log_files', self)
+
+    class << self
+      def transaction(config, &control)
+        oauth = config.param('oauth', :hash)
+        task = {
+          'login_url' => config.param('login_url', :string, default: 'https://login.salesforce.com'),
+          'oauth_client_id' => oauth['id'],
+          'oauth_client_secret' => oauth['secret'],
+          'username' => config.param('username', :string),
+          'password' => config.param('password', :string),
+          'last_log_date' => config.param('last_log_date', :string, default: '0001-01-01T00:00:00Z'),
+          'max_retry_times' => config.param('max_retry_times', :integer, default: 2),
+        }
+        threads = config.param('threads', :integer, default: 2)
+        task['client'] = client = HTTPClient.new
+
+        begin
+          oauth(task)
+          task['records'] = query(task)
+
+          reports = yield(task, [], threads)
+
+          last_log_date_report = reports.max_by { |report|
+            report['last_log_date']
+          }
+          config.merge(last_log_date_report)
+        rescue
+          # TODO: log
+          raise
+        end
+      end
+
+      def run(task, schema, index, page_builder)
+        # This input plugin does not use schema
+        new(task, index, page_builder).run
+      end
+
+    private
+
+      def oauth(task)
+        client = task['client']
+        params = {
+          :grant_type => 'password',
+          :client_id => task['oauth_client_id'],
+          :client_secret => task['oauth_client_secret'],
+          :username => task['username'],
+          :password => task['password']
+        }
+        with_retry(task) {
+          res = client.post(task['login_url'] + '/services/oauth2/token', params, :Accept => 'application/json; charset=UTF-8')
+          parsed = JSON.parse(res.body)
+          client.base_url = parsed['instance_url']
+          client.default_header = { 'Authorization' => 'Bearer ' + parsed['access_token'] }
+          nil
+        }
+      end
+
+      def query(task)
+        client = task['client']
+        query = "Select LogDate, EventType, LogFile from EventLogFile Where LogDate > #{task['last_log_date']}"
+        with_retry(task) {
+          res = client.get('/services/data/v32.0/query/', :q => query)
+          JSON.parse(res.body)['records']
+        }
+      end
+
+      def with_retry(task)
+        retry_times = 0
+        begin
+          yield
+        rescue
+          # TODO: log
+          retry_times += 1
+          retry if retry_times < task['max_retry_times']
+          raise
+        end
+      end
+    end
+
+    def initialize(task, index, page_builder)
+      @page_builder = page_builder
+      @records = task['records']
+      @last_log_date = Time.parse(task['last_log_date'])
+      @client = task['client']
+    end
+
+    def run
+      @records.each do |record|
+        event_type = record['EventType']
+        @last_log_date = [@last_log_date, Time.parse(record['LogDate']).to_i].max
+        log_file = record['LogFile']
+        CSV.parse(@client.get_content(log_file), headers: true) do |row|
+          row['time'] = Time.parse(row['TIMESTAMP']).to_i
+          @page_builder.add(row)
+        end
+      end
+      @page_builder.finish unless @records.empty?
+
+      commit_report = {
+        'last_log_date' => @last_log_date.xmlschema
+      }
+      commit_report
+    end
+  end
+
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: embulk-plugin-input-sfdc-event-log-files
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hiroshi Nakamura
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Embulk plugin for Salesforce.com Event Log Files input
+email: nahi@ruby-lang.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/embulk/input_sfdc_event_log_files.rb
+- README.md
+homepage: https://github.com/nahi/
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Embulk plugin for Salesforce.com Event Log Files input
+test_files: []
+has_rdoc: