embulk-input-slack-history 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 49031b2f1709e7d4b70fa643310e63f539372b3b
+  data.tar.gz: 211c4b5a59e0e9d21f3daa264aba4232aa8c6c3c
+SHA512:
+  metadata.gz: 5432efc108632885e6948f3ccaf6e02c0994f207287b677af434a1ce80a61c4a1434c9c0c88af4e20a2ad189f14e06703e888e7c1072d7e6173d273f7fdddb02
+  data.tar.gz: ee30df521d35f41ad279ef22d15c51b2cda47786609dc92dc6abc26ed54a3cb72187e5c11e6174072abac928418f0ca71fca65ccdbe04cb29d7e236b0eddefee

data/ChangeLog

@@ -0,0 +1,16 @@
+
+2015-02-09 version 0.0.1:
+
+* The first release
+
+
+2015-02-16 version 0.0.2:
+
+* Directory tree changed for embulk v0.4.1
+
+
+2015-02-17 version 0.1.0:
+
+* Change name for embulk v0.4.1
+* Refactoring code 
+

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    embulk-input-slack-history (0.1.0)
+      rest-client (>= 1.7.2)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    rest-client (1.7.2)
+    rake (10.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  embulk-input-slack-history!
+  rake (>= 0.9.2)

data/README.md

@@ -0,0 +1,33 @@
+# embulk-input-slack-history
+
+This [Embulk](https://github.com/embulk/embulk) input plugin for Slack chat history.
+
+
+## Installation
+
+		$ java -jar embulk.jar gem install embulk-input-slack-history
+
+## Configuration
+
+- **token** Slack API token 'Generate from https://api.slack.com/' (string, required)
+- **continuous** last time read continuation (string, default: false)
+- **filepath** continuous information file save path (string, default: /tmp)
+
+### Example
+
+```yaml
+in:
+  type: slack_history
+  token: slackapitoken
+  continuous: true # optional
+  filepath: /tmp # optional
+out:
+  type: stdout
+```
+
+## TODO
+
+- attachment file download
+- do not update *.oldest file at preview time
+- error handling and resume
+- multi thread execute Slack API if needed

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :default => [:build]

data/embulk-input-slack-history.gemspec

@@ -0,0 +1,21 @@
+
+Gem::Specification.new do |gem|
+  gem.name          = "embulk-input-slack-history"
+  gem.version       = "0.1.0"
+
+  gem.summary       = %q{Embulk input plugin for Slack chat history}
+  gem.description   = gem.summary
+  gem.authors       = ["Akihiro YAGASAKI"]
+  gem.email         = ["yaggytter@momiage.com"]
+  gem.license       = "Apache 2.0"
+  gem.homepage      = "https://github.com/yaggytter/embulk-input-slack-history"
+
+  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.add_dependency 'rest-client', ['>= 1.7.2']
+  gem.add_development_dependency 'bundler', ['~> 1.0']
+  gem.add_development_dependency 'rake', ['>= 0.9.2']
+end

data/lib/embulk/input/slack_history.rb

@@ -0,0 +1,246 @@
+module Embulk
+  module Plugin
+
+    require 'json'
+    require 'rest-client'
+
+    class SlackApiException < Exception; end
+
+    class SlackApi
+
+      def initialize(token)
+        @token = token
+        @members = {}
+        @groups = {}
+        @channels = {}
+      end
+
+      def pre()
+
+        # TODO: need implement error handling
+        # HTTP Status Code
+        # RestClient Exception
+        # result is NOT 'ok'
+
+        json = RestClient.get('https://slack.com/api/channels.list', {:params => {'token' => @token, 'pretty' => 1}})
+        result = JSON.parse(json)
+
+        if !result['ok'] then
+          raise SlackApiException
+        end
+
+        result["channels"].each{|channel|
+          @channels[channel["id"]] = channel["name"]
+        }
+
+        json = RestClient.get('https://slack.com/api/groups.list', {:params => {'token' => @token, 'pretty' => 1}})
+        result = JSON.parse(json)
+
+        if !result['ok'] then
+          raise SlackApiException
+        end
+
+        result["groups"].each{|group|
+          @groups[group["id"]] = group["name"]
+        }
+
+        json = RestClient.get('https://slack.com/api/users.list', {:params => {'token' => @token, 'pretty' => 1}})
+        result = JSON.parse(json)
+
+        if !result['ok'] then
+          raise SlackApiException
+        end
+
+        result["members"].each{|member|
+          @members[member["id"]] = member["name"]
+        }
+
+        return true
+
+      end
+      
+      def parse_history(message)
+
+        res = {}
+
+        res["ts"] = message["ts"]
+        res["username"] = @members[message["user"]]
+        res["userid"] = message["user"]
+        res["message"] = message["text"]
+
+        return res
+
+      end
+
+      def get_oldest_filepath(filepath, channelid)
+        return filepath + "/" + channelid + ".oldest"
+      end
+
+      def get_continuous_param(continuous, filepath, channelid)
+
+        if !continuous then
+          return ""
+        end
+
+        param = ""
+
+        fullpath = get_oldest_filepath(filepath, channelid)
+
+        if !File.exist?(fullpath) then
+          return ""
+        end        
+
+        line = File.read(fullpath)
+
+        param = "&oldest=" + line + "&inclusive=0"
+
+        return param
+
+      end
+
+      def update_oldest(continuous, filepath, channelid, newest)
+
+        if !continuous || newest == 0.0 then
+          return
+        end        
+
+        fullpath = get_oldest_filepath(filepath, channelid)
+        File.write(fullpath, newest)
+
+      end
+
+      def get_history_by_channels(continuous, filepath, channels, isprivate)
+
+        res = []
+        i = 0
+
+        channels.each{|id, name|
+
+          api = "channels"
+          if isprivate == "yes" then
+            api = "groups"
+          end
+
+          param = get_continuous_param(continuous, filepath, id)
+
+          json = RestClient.get('https://slack.com/api/' + api + '.history', {:params => {'token' => @token, 'channel' => id, 'pretty' => 1}})
+          result = JSON.parse(json)
+
+          newest = 0.0
+
+          result["messages"].each{|message|
+
+            mes = parse_history(message)
+            mes["channelid"] = id
+            mes["channelname"] = name
+            mes["private"] = isprivate
+            res[i] = mes
+            i += 1
+
+            ts = message['ts'].to_f
+            if newest < ts then
+              newest = ts
+            end
+
+          }
+
+          update_oldest(continuous, filepath, id, newest)
+
+        }
+
+        return res
+
+      end
+
+      def get_history(continuous, filepath)
+
+        res = get_history_by_channels(continuous, filepath, @channels, "no")
+        res.concat(get_history_by_channels(continuous, filepath, @groups, "yes"))
+
+        return res
+
+      end
+
+
+    end
+
+    class InputSlackHistory < InputPlugin
+
+      # input plugin file name must be: embulk/input_<name>.rb
+      Plugin.register_input('slack_history', self)
+
+      def self.transaction(config, &control)
+
+        @noerror = true
+
+        threads = 1
+
+        token = config.param('token', :string)
+        continuous = config.param('continuous', :bool, default: false)
+        filepath = config.param('filepath', :string, default: '/tmp')
+        preview = config.param('preview', :string, default: 'no')
+
+        task = {
+          'token' => token,
+          'continuous' => continuous,
+          'filepath' => filepath,
+          'preview' => preview
+        }
+
+        columns = [
+          Column.new(0, 'channelid', :string),
+          Column.new(1, 'channelname', :string),
+          Column.new(2, 'private', :string),
+          Column.new(3, 'datetime', :timestamp),
+          Column.new(4, 'username', :string),
+          Column.new(5, 'userid', :string),
+          Column.new(6, 'message', :string),
+        ]
+
+        puts "Slack history input started."
+        commit_reports = yield(task, columns, threads)
+        puts "Slack history input finished."
+
+        next_config_diff = {}
+        return next_config_diff
+
+      end
+
+
+      def initialize(task, schema, index, page_builder)
+        super
+        @slack = SlackApi.new(task['token'])
+        @noerror = @slack.pre()
+      end
+
+      def run
+
+        if !@noerror then
+          @page_builder.finish
+          return
+        end
+
+        messages = @slack.get_history(@task['continuous'], @task['filepath'])
+
+        messages.each{|message|
+          @page_builder.add([
+            message['channelid'],
+            message['channelname'],
+            message['private'],
+            Time.at(message['ts'].to_f),
+            message['username'],
+            message['userid'],
+            message['message']
+            ])
+        }
+
+        @page_builder.finish
+
+        commit_report = {}
+        return commit_report
+      end
+
+    end
+
+  end
+end

data/slack_history_configsample.yml

@@ -0,0 +1,7 @@
+in:
+  type: slack_history
+  token: slackapitoken
+  continuous: true # optional
+  filepath: /tmp # optional
+out:
+  type: stdout

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-slack-history
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Akihiro YAGASAKI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Embulk input plugin for Slack chat history
+email:
+- yaggytter@momiage.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ChangeLog
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- embulk-input-slack-history.gemspec
+- lib/embulk/input/slack_history.rb
+- slack_history_configsample.yml
+homepage: https://github.com/yaggytter/embulk-input-slack-history
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Embulk input plugin for Slack chat history
+test_files: []