slack_scratcher 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bed4b833ec200a7e4269205379ff12118ee364c9
+  data.tar.gz: 1bf60094a6d36fc79843f6ed0cecf6670d1ea78e
+SHA512:
+  metadata.gz: df7817192ebf16acc8fac4dcea7029d2f8bfb44c3d95dbcff8606913000802b281f46d511aa242907908ea7bb5c876d24e67b4ab8a5a5e1c0595251525853ff2
+  data.tar.gz: 335bea07ac31c3252cce066f3cc8fba41e75e39153652d469d47d34d9a6d115b6931d99f54d296bda0328e1dcb92514a2e982c9f7c170852f9e9cb93fa8010d9

data/lib/slack_scratcher.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+require 'logger'
+
+module SlackScratcher
+  autoload :Model, 'slack_scratcher/model'
+  autoload :Loader, 'slack_scratcher/loader'
+  autoload :Adapter, 'slack_scratcher/adapter'
+  autoload :Router, 'slack_scratcher/router'
+  autoload :Collector, 'slack_scratcher/collector'
+  autoload :Error, 'slack_scratcher/error'
+
+  def self.logger
+    @logger ||= ::Logger.new(STDOUT)
+  end
+end

data/lib/slack_scratcher/adapter.rb

@@ -0,0 +1,6 @@
+module SlackScratcher
+  module Adapter
+    autoload :Elasticsearch, 'slack_scratcher/adapter/elasticsearch'
+    autoload :File, 'slack_scratcher/adapter/file'
+  end
+end

data/lib/slack_scratcher/adapter/elasticsearch.rb

@@ -0,0 +1,82 @@
+require 'elasticsearch'
+
+module SlackScratcher
+  module Adapter
+    class Elasticsearch
+      attr_reader :client
+
+      def initialize(hosts, metadata)
+        @client = ::Elasticsearch::Client.new hosts: hosts
+        @metadata = metadata
+      end
+
+      def send(raw_data)
+        data = format_bulk(raw_data)
+        @client.bulk data unless raw_data.empty?
+      rescue ::Elasticsearch::Transport::Transport::Errors::BadRequest => error
+        puts error
+      end
+
+      def timestamp_of_last_channel_log(channel_name)
+        
+      end
+
+      def ready_index
+        unless index?
+          create_index
+          put_mapping
+        end
+      end
+
+      private
+
+      def index
+        @metadata[:index]
+      end
+
+      def type
+        @metadata[:type]
+      end
+
+      def index?
+        @client.indices.exists(index: index)
+      end
+
+      def create_index
+        @client.indices.create(index: index)
+      end
+
+      def put_mapping
+        @client.indices.put_mapping create_body(mapping)
+      end
+
+      def mapping
+        { type =>
+          { '_timestamp' => { 'enabled' => true, 'path' => 'dataetime' },
+            '_id' => { 'path' => 'uid' } } }
+      end
+
+      def create_body(body = {})
+        {
+          index: index,
+          type: type,
+          body: body
+        }
+      end
+
+      def format_bulk(data)
+        { body: data.map { |log| format_log(log) } }
+      end
+
+      def format_log(log)
+        { index:
+            {
+              _index: index,
+              _type: type,
+              data: log
+            }
+        }
+      end
+    end
+  end
+end

data/lib/slack_scratcher/adapter/file.rb

@@ -0,0 +1,7 @@
+module SlackScratcher
+  module Adapter
+    class File
+
+    end
+  end
+end

data/lib/slack_scratcher/collector.rb

@@ -0,0 +1,16 @@
+module SlackScratcher
+  class Collector
+    def initialize(loader, adapter)
+      @loader = loader
+      @adapter = adapter
+    end
+
+    def collect_loop
+      loop do
+        @loader.each(@adapter) do |data|
+          @adapter.send data
+        end
+      end
+    end
+  end
+end

data/lib/slack_scratcher/error.rb

@@ -0,0 +1,7 @@
+module SlackScratcher
+  module Error
+    autoload :UserNotFoundError, 'slack_scratcher/error/user_not_found_error'
+    autoload :SlackApiError, 'slack_scratcher/error/slack_api_error'
+    autoload :FileNotFound, 'slack_scratcher/error/file_not_found'
+  end
+end

data/lib/slack_scratcher/error/file_not_found.rb

@@ -0,0 +1,6 @@
+module SlackScratcher
+  module Error
+    class FileNotFound < StandardError
+    end
+  end
+end

data/lib/slack_scratcher/error/slack_api_error.rb

@@ -0,0 +1,6 @@
+module SlackScratcher
+  module Error
+    class SlackApiError < StandardError
+    end
+  end
+end

data/lib/slack_scratcher/error/user_not_found_error.rb

@@ -0,0 +1,6 @@
+module SlackScratcher
+  module Error
+    class UserNotFoundError < StandardError
+    end
+  end
+end

data/lib/slack_scratcher/loader.rb

@@ -0,0 +1,6 @@
+module SlackScratcher
+  module Loader
+    autoload :File, 'slack_scratcher/loader/file'
+    autoload :Api, 'slack_scratcher/loader/api'
+  end
+end

data/lib/slack_scratcher/loader/api.rb

@@ -0,0 +1,65 @@
+require 'slack'
+
+module SlackScratcher
+  module Loader
+    class Api
+      include Enumerable
+
+      WAIT_TIME = 15
+
+      def initalize
+        
+      end
+
+      def each(adapter)
+        active_channels do |channel|
+          wait
+          from = adapter.time_of_channel_last_log(channel.name)
+          to = 'now'
+          yield channel_history(channel, from, to)
+        end
+      end
+
+      private
+
+      def channels
+        result = Slack.channels_list['channels']
+        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+
+        result.map { |ch| { id: ch['id'], name: ch['name'] } }
+      end
+
+      def active_channels
+        result = Slack.channels_list['channels']
+        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+
+        result.select { |ch| ch['is_archived'] == false }
+          .map { |ch| { id: ch['id'], name: ch['name'] } }
+      end
+
+      def user_info(user_id)
+        result = Slack.users_info(user: user_id)
+        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+
+        result
+      end
+
+      def channel_history(channel, from, to = Time.now)
+        attrs = {
+          channel: channel,
+          oldest: from.to_i,
+          latest: to.to_i
+        }
+
+        result = Slack.channels_history(attrs)
+        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+
+        result
+      end
+
+      def wait
+        sleep WAIT_TIME
+      end
+    end
+  end
+end

data/lib/slack_scratcher/loader/file.rb

@@ -0,0 +1,77 @@
+require 'oj'
+
+module SlackScratcher
+  module Loader
+    class File
+      include Enumerable
+
+      def initialize(target_dir)
+        fail ArgumentError unless ::File.directory?(target_dir)
+
+        @target = target_dir
+        @users = users
+        @channels = channels
+      end
+
+      def each
+        files.each do |file|
+          yield parse_log_file(file), file
+        end
+
+        true
+      end
+
+      private
+
+      def get_channel_dir(path)
+        ::File.dirname(path).split('/').last
+      end
+
+      def users
+        load "#{@target}/users.json", 'id'
+      end
+
+      def channels
+        load "#{@target}/channels.json", 'name'
+      end
+
+      def load(target, index_column)
+        fail SlackScratcher::Error::FileNotFound unless ::File.exist? target
+
+        channels = Oj.load(::File.read(target))
+        index_data channels, index_column
+      end
+
+      def index_data(dataset, column)
+        dataset.map { |data| { data[column] => data } }.inject({}, :merge)
+      end
+
+      def channel_info(log_file)
+        name = get_channel_dir(log_file)
+        { name: name, id: @channels[name]['id'] }
+      end
+
+      def parse_log_file(log_file)
+        channel = channel_info(log_file)
+        logs = Oj.load(::File.read(log_file))
+        chats = SlackScratcher::Model::Chats.new(logs, channel, @users)
+
+        chats.refined_data
+      end
+
+      def files
+        channels.inject([]) do |arr, channel|
+          arr + log_files(channel)
+        end
+      end
+
+      def channels
+        Dir["#{@target}/*/"]
+      end
+
+      def log_files(channel)
+        Dir["#{channel}*.json"]
+      end
+    end
+  end
+end

data/lib/slack_scratcher/model.rb

@@ -0,0 +1,5 @@
+module SlackScratcher
+  module Model
+    autoload :Chats, 'slack_scratcher/model/chats'
+  end
+end

data/lib/slack_scratcher/model/chats.rb

@@ -0,0 +1,64 @@
+module SlackScratcher
+  module Model
+    class Chats
+      attr_reader :refined_data, :data
+
+      def initialize(data, channel, users)
+        if !data.is_a?(Array) || !users.is_a?(Hash) || !channel.is_a?(Hash)
+          fail ArgumentError
+        end
+
+        @data = data
+        @users = users
+        @channel = channel
+
+        @refined_data = refine
+      end
+
+      private
+
+      def refine
+        @data
+          .map { |log| refine_data(log) }
+          .select { |log| !log['uid'].nil? }
+      end
+
+      def refine_data(log)
+        user = find_user(log['user']) unless user
+
+        log['name'] = user['name']
+        log['profile_image'] = user['profile']['image_32']
+        log['text'] = refine_text(log['text'])
+        log['channel'] = @channel[:name]
+        log['channel_id'] = @channel[:id]
+        log['datetime'] = Time.at(log['ts'].to_f).iso8601
+
+        log['uid'] = create_uid(log)
+
+        log
+      rescue SlackScratcher::Error::UserNotFoundError
+        user = { 'user' => 'undefined' }
+        user['profile'] = { 'image_32' => '' }
+      end
+
+      def create_uid(log)
+        "#{log['datetime']}-#{log['channel_id']}-#{log['user']}"
+      end
+
+      def refine_text(text)
+        text = ":#{text}" if text.is_a?(Symbol)
+
+        text
+          .gsub(%r{<@([A-Z0-9]{3,10})>}) { '@' + @users[$1]['name'] }
+          .gsub(%r{<(http(s)?://.*?)>}) { $1 }
+      end
+
+      def find_user(user)
+        result = @users[user]
+
+        fail SlackScratcher::Error::UserNotFoundError if result.nil?
+        result
+      end
+    end
+  end
+end

data/lib/slack_scratcher/router.rb

@@ -0,0 +1,17 @@
+module SlackScratcher
+  class Router
+    def initialize(loader, adapter)
+      @loader = loader
+      @adapter = adapter
+    end
+
+    def route
+      @adapter.ready_index
+
+      @loader.each do |data, file|
+        @adapter.send data
+        SlackScratcher.logger.info "* #{file} is routed."
+      end
+    end
+  end
+end

data/lib/slack_scratcher/version.rb

@@ -0,0 +1,3 @@
+module SlackScratcher
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: slack_scratcher
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Daekwon Kim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: slack-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Importing slack log from exported files or API to elasticsearch or other
+  datastores
+email:
+- propellerheaven@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/slack_scratcher.rb
+- lib/slack_scratcher/adapter.rb
+- lib/slack_scratcher/adapter/elasticsearch.rb
+- lib/slack_scratcher/adapter/file.rb
+- lib/slack_scratcher/collector.rb
+- lib/slack_scratcher/error.rb
+- lib/slack_scratcher/error/file_not_found.rb
+- lib/slack_scratcher/error/slack_api_error.rb
+- lib/slack_scratcher/error/user_not_found_error.rb
+- lib/slack_scratcher/loader.rb
+- lib/slack_scratcher/loader/api.rb
+- lib/slack_scratcher/loader/file.rb
+- lib/slack_scratcher/model.rb
+- lib/slack_scratcher/model/chats.rb
+- lib/slack_scratcher/router.rb
+- lib/slack_scratcher/version.rb
+homepage: http://slack-scratcher.nacyot.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Slack log import tool
+test_files: []
+has_rdoc: