embulk-plugin-input-sfdc-event-log-files 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82ff6152830fd1e9fe317091a30325d28e322848
-  data.tar.gz: dd78b5bbb4e11d215be0570023304385225ddf4f
+  metadata.gz: 26d4a69db8632348b043774556a10b3c7bbff8e3
+  data.tar.gz: a2a7cfd359eb9a89625a4cc0f1bdd0229f236fc3
 SHA512:
-  metadata.gz: 3cd3a3c7acbe65789601d8b158cfbde4d37528e406187f9d43ddebbd3144760e0f434377497e1675c9df3e1d86a2a570e5731ef996d910c065e19e0e8e83332c
-  data.tar.gz: 38118c1e77adf1d538e1bf3b82a710427f52192ab331f3c90230c538d22f3def32216efa274527aee8b1a078a4148a6e5a04631562737fe236e758d158c43e0c
+  metadata.gz: d54660a917a3d419f3adef4a7f3cd9e3f7cd155323a04c3b26a6e3cdd0746c518793c1cca543a48caf9f0e5901cfa7b451a8eaefd84bd416a563aefdae5a8273
+  data.tar.gz: 51b9b0c0724aa171876c611af1df816535a4a267dfb8286484413b97f05238693899bc11d277c8242ae5e956aceed1bc6087ecf4d049622b09e3860a69dd394d

data/README.md

@@ -3,4 +3,6 @@ embulk-plugin-input-sfdc-event-log-files
 
 Embulk plugin for Salesforce.com Event Log Files input
 
-To be written...
+At this moment this plugin is a sample implementation of InputPlugin.
+Without GuessPlugin implementation you cannot create proper schema configuration.
+Stay tuned.

data/lib/embulk/input_sfdc_event_log_files.rb

@@ -20,17 +20,23 @@ module Embulk
           'last_log_date' => config.param('last_log_date', :string, default: '0001-01-01T00:00:00Z'),
           'max_retry_times' => config.param('max_retry_times', :integer, default: 2),
         }
-        idx = -1
-        schema = config.param('schema', :array).map { |c| idx += 1; Column.new(idx, c['name'], c['type'].to_sym) }
         threads = config.param('threads', :integer, default: 2)
-        task['client'] = client = HTTPClient.new
+        idx = -1
+        schema = config.param('schema', :array, default: []).map { |c| idx += 1; Column.new(idx, c['name'], c['type'].to_sym) }
 
         begin
-          oauth(task)
-          task['records'] = query(task)
+          client = HTTPClient.new
+          parsed = oauth(client, task)
+          task['instance_url'] = parsed['instance_url']
+          task['access_token'] = parsed['access_token']
+          init_client(client, task)
 
-          reports = yield(task, schema, threads)
+          task['records'] = query(client, task)
 
+          if schema.empty?
+            raise 'empty schema given'
+          end
+          reports = yield(task, schema, threads)
           last_log_date_report = reports.max_by { |report|
             report['last_log_date']
           }
@@ -41,10 +47,15 @@ module Embulk
         end
       end
 
+      def init_client(client, task)
+        client.base_url = task['instance_url']
+        client.default_header = { 'Authorization' => 'Bearer ' + task['access_token'] }
+        client
+      end
+
     private
 
-      def oauth(task)
-        client = task['client']
+      def oauth(client, task)
         params = {
           :grant_type => 'password',
           :client_id => task['oauth_client_id'],
@@ -54,15 +65,11 @@ module Embulk
         }
         with_retry(task) {
           res = client.post(task['login_url'] + '/services/oauth2/token', params, :Accept => 'application/json; charset=UTF-8')
-          parsed = JSON.parse(res.body)
-          client.base_url = parsed['instance_url']
-          client.default_header = { 'Authorization' => 'Bearer ' + parsed['access_token'] }
-          nil
+          JSON.parse(res.body)
         }
       end
 
-      def query(task)
-        client = task['client']
+      def query(client, task)
         query = "Select LogDate, EventType, LogFile from EventLogFile Where LogDate > #{task['last_log_date']}"
         with_retry(task) {
           res = client.get('/services/data/v32.0/query/', :q => query)
@@ -83,30 +90,34 @@ module Embulk
       end
     end
 
-    def initialize(task, schema, index, page_builder)
-      @schema = schema
-      @page_builder = page_builder
-      @records = task['records']
-      @last_log_date = Time.parse(task['last_log_date'])
-      @client = task['client']
-    end
+    attr_reader :task
+    attr_reader :schema
+    attr_reader :page_builder
 
     def run
-      columns = @schema.map { |c| c.name }
-      @page_builder.finish
-      @records.each do |record|
+      client = self.class.init_client(HTTPClient.new, task)
+      records = task['records']
+      last_log_date = Time.parse(task['last_log_date'])
+      columns = schema.map { |c| c.name }
+      records.each do |record|
         event_type = record['EventType']
-        @last_log_date = [@last_log_date, Time.parse(record['LogDate']).to_i].max
+        last_log_date = [last_log_date, Time.parse(record['LogDate']).to_i].max
         log_file = record['LogFile']
-        CSV.parse(@client.get_content(log_file), headers: true) do |row|
-          row['TIMESTAMP'] = Time.parse(row['TIMESTAMP']).to_i
-          @page_builder.add(row.to_hash.values_at(*columns))
+        log_body = client.get_content(log_file)
+        CSV.parse(log_body, headers: true) do |row|
+          if row['TIMESTAMP']
+            begin
+              row['TIMESTAMP'] = Time.parse(row['TIMESTAMP']).to_i
+            rescue
+              # ignore
+            end
+          end
+          page_builder.add(row.to_hash.values_at(*columns).map(&:to_s))
         end
       end
-      @page_builder.finish unless @records.empty?
-
+      page_builder.finish unless records.empty?
       commit_report = {
-        'last_log_date' => @last_log_date.xmlschema
+        'last_log_date' => last_log_date.xmlschema
       }
       commit_report
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-plugin-input-sfdc-event-log-files
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Hiroshi Nakamura