slack_scratcher 0.0.1 → 0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bed4b833ec200a7e4269205379ff12118ee364c9
-  data.tar.gz: 1bf60094a6d36fc79843f6ed0cecf6670d1ea78e
+  metadata.gz: 56b9028533b252ce83872884a332a85d344f7dc0
+  data.tar.gz: 670049b263e5c25aa3222311033546af65dfce6b
 SHA512:
-  metadata.gz: df7817192ebf16acc8fac4dcea7029d2f8bfb44c3d95dbcff8606913000802b281f46d511aa242907908ea7bb5c876d24e67b4ab8a5a5e1c0595251525853ff2
-  data.tar.gz: 335bea07ac31c3252cce066f3cc8fba41e75e39153652d469d47d34d9a6d115b6931d99f54d296bda0328e1dcb92514a2e982c9f7c170852f9e9cb93fa8010d9
+  metadata.gz: a65adf9dc0a91ad769aabfc1b7cf82133c95f19de544f3f368cdf20ab1444c2986e46928e0f1689dc05f8f0651da514c63174a71d1be2b12bff56b904247f408
+  data.tar.gz: 3ecb3d72f1f036f4c13ae7e706c82c094d75930c2d89d7ee5f8c657e6c0dd9052142df94bb831d021ca8264682825f66a141a981cb4c7982ac89b767eb79b7e4

data/lib/slack_scratcher/adapter/elasticsearch.rb

@@ -18,7 +18,11 @@ module SlackScratcher
       end
 
       def timestamp_of_last_channel_log(channel_name)
-        
+        request_body = create_body(query_for_last_log(channel_name))
+        log = @client.search request_body
+
+        return 0 if log['hits']['total'] == 0
+        log['hits']['hits'][0]['_source']['ts']
       end
 
       def ready_index
@@ -51,9 +55,34 @@ module SlackScratcher
       end
 
       def mapping
-        { type =>
-          { '_timestamp' => { 'enabled' => true, 'path' => 'dataetime' },
-            '_id' => { 'path' => 'uid' } } }
+        {
+          type => {
+            '_timestamp' => {
+              'enabled' => true,
+              'path' => 'dataetime'
+            },
+            '_id' => {
+              'path' => 'uid'
+            },
+            '_routing' => {
+              'required' => true,
+              'path' => 'channel_id'
+            }
+          }
+        }
+      end
+
+      def query_for_last_log(channel_name)
+        {
+          size: 1,
+          sort:
+            [ { datetime: { order: 'desc' } }],
+          query: {
+            match: {
+              channel: channel_name
+            }
+          }
+        }
       end
 
       def create_body(body = {})

data/lib/slack_scratcher/error/token_not_set.rb

@@ -0,0 +1,6 @@
+module SlackScratcher
+  module Error
+    class TokenNotSet < StandardError
+    end
+  end
+end

data/lib/slack_scratcher/error.rb

@@ -3,5 +3,6 @@ module SlackScratcher
     autoload :UserNotFoundError, 'slack_scratcher/error/user_not_found_error'
     autoload :SlackApiError, 'slack_scratcher/error/slack_api_error'
     autoload :FileNotFound, 'slack_scratcher/error/file_not_found'
+    autoload :TokenNotSet, 'slack_scratcher/error/token_not_set'
   end
 end

data/lib/slack_scratcher/helper.rb

@@ -0,0 +1,8 @@
+module SlackScratcher
+  module Helper
+    def self.index_data(dataset, column)
+      dataset.map { |data| { data[column] => data } }.inject({}, :merge)
+    end
+  end
+end
+

data/lib/slack_scratcher/loader/api.rb

@@ -5,56 +5,95 @@ module SlackScratcher
     class Api
       include Enumerable
 
-      WAIT_TIME = 15
+      WAIT_TIME = 1
 
-      def initalize
-        
+      def initialize(token = nil)
+        authenticate_slack(token)
       end
 
       def each(adapter)
-        active_channels do |channel|
-          wait
-          from = adapter.time_of_channel_last_log(channel.name)
-          to = 'now'
-          yield channel_history(channel, from, to)
+        @users || set_users
+
+        active_channels.each do |channel|
+          from = adapter.timestamp_of_last_channel_log(channel[:name])
+          yield parse_log(channel, from), channel
         end
+
+        true
       end
 
       private
 
-      def channels
-        result = Slack.channels_list['channels']
-        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+      def set_users
+        @users = users
+        SlackScratcher.logger.info "* Users list refreshed"
+      end
+
+      def check_users(logs)
+        logs
+          .select { |log| log.key? 'user' }
+          .map { |log| log['user'] }
+          .any? { |user| @users[user].nil? }
+      end
+
+      def parse_log(channel, from)
+        logs = channel_history(channel[:id], from)
+        if check_users(logs)
+          set_users
+        end
+        SlackScratcher::Model::Chats.new(logs, channel, @users).refined_data
+      end
 
-        result.map { |ch| { id: ch['id'], name: ch['name'] } }
+      def authenticate_slack(token)
+        token ||= ENV['SLACK_TOKEN']
+        fail SlackScratcher::Error::TokenNotSet unless token
+        Slack.configure { |config| config.token = token }
+      end
+
+      def channels
+        response = validate_response(Slack.channels_list)
+        index_channels response['channels']
       end
 
       def active_channels
-        result = Slack.channels_list['channels']
-        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+        wait
 
-        result.select { |ch| ch['is_archived'] == false }
-          .map { |ch| { id: ch['id'], name: ch['name'] } }
+        response = validate_response(Slack.channels_list)
+        index_channels filter_active_channels(response['channels'])
       end
 
-      def user_info(user_id)
-        result = Slack.users_info(user: user_id)
-        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+      def filter_active_channels(data)
+        data.select { |channel| channel['is_archived'] == false }
+      end
 
-        result
+      def index_channels(data)
+        data.map { |channel| { id: channel['id'], name: channel['name'] } }
       end
 
-      def channel_history(channel, from, to = Time.now)
+      def users
+        wait
+
+        user_list = validate_response(Slack.users_list, :members)
+        SlackScratcher::Helper.index_data user_list, 'id'
+      end
+
+      def channel_history(channel_id, from, to = Time.now)
+        wait
+
         attrs = {
-          channel: channel,
-          oldest: from.to_i,
-          latest: to.to_i
+          channel: channel_id,
+          oldest: from,
+          latest: to,
+          count: 1000
         }
 
-        result = Slack.channels_history(attrs)
-        fail SlackScratcher::Error::ApiError unless result['ok'] == true
+        validate_response Slack.channels_history(attrs), :messages
+      end
 
-        result
+      def validate_response(response, key = nil)
+        fail SlackScratcher::Error::ApiError unless response['ok'] == true
+        return response unless key
+        response[key.to_s]
       end
 
       def wait

data/lib/slack_scratcher/loader/file.rb

@@ -13,7 +13,7 @@ module SlackScratcher
         @channels = channels
       end
 
-      def each
+      def each(_ = nil)
         files.each do |file|
           yield parse_log_file(file), file
         end
@@ -39,11 +39,7 @@ module SlackScratcher
         fail SlackScratcher::Error::FileNotFound unless ::File.exist? target
 
         channels = Oj.load(::File.read(target))
-        index_data channels, index_column
-      end
-
-      def index_data(dataset, column)
-        dataset.map { |data| { data[column] => data } }.inject({}, :merge)
+        SlackScratcher::Helper.index_data channels, index_column
       end
 
       def channel_info(log_file)
@@ -54,18 +50,16 @@ module SlackScratcher
       def parse_log_file(log_file)
         channel = channel_info(log_file)
         logs = Oj.load(::File.read(log_file))
-        chats = SlackScratcher::Model::Chats.new(logs, channel, @users)
-
-        chats.refined_data
+        SlackScratcher::Model::Chats.new(logs, channel, @users).refined_data
       end
 
       def files
-        channels.inject([]) do |arr, channel|
+        channel_dirs.inject([]) do |arr, channel|
           arr + log_files(channel)
         end
       end
 
-      def channels
+      def channel_dirs
         Dir["#{@target}/*/"]
       end
 

data/lib/slack_scratcher/model/chats.rb

@@ -24,9 +24,9 @@ module SlackScratcher
       end
 
       def refine_data(log)
-        user = find_user(log['user']) unless user
+        user = find_user(log) unless user
 
-        log['name'] = user['name']
+        log['username'] = user['name']
         log['profile_image'] = user['profile']['image_32']
         log['text'] = refine_text(log['text'])
         log['channel'] = @channel[:name]
@@ -36,13 +36,10 @@ module SlackScratcher
         log['uid'] = create_uid(log)
 
         log
-      rescue SlackScratcher::Error::UserNotFoundError
-        user = { 'user' => 'undefined' }
-        user['profile'] = { 'image_32' => '' }
       end
 
       def create_uid(log)
-        "#{log['datetime']}-#{log['channel_id']}-#{log['user']}"
+        "#{log['datetime']}-#{log['channel_id']}-#{log['username']}"
       end
 
       def refine_text(text)
@@ -53,11 +50,27 @@ module SlackScratcher
           .gsub(%r{<(http(s)?://.*?)>}) { $1 }
       end
 
-      def find_user(user)
-        result = @users[user]
+      def find_user(log)
+        return bot_user(log) if log.key?('username')
+
+        result = @users[log['user']]
 
         fail SlackScratcher::Error::UserNotFoundError if result.nil?
         result
+      rescue SlackScratcher::Error::UserNotFoundError
+        unknown_user
+      end
+
+      def undefined_user
+        user = { 'name' => '_unknown_' }
+        user['profile'] = { 'image_32' => '' }
+        user
+      end
+
+      def bot_user(log)
+        user = { 'name' => log['username'] }
+        user['profile'] = { 'image_32' => log['icons']['image_48'] }
+        user
       end
     end
   end

data/lib/slack_scratcher/router.rb

@@ -6,11 +6,29 @@ module SlackScratcher
     end
 
     def route
+      ready
+      _route
+    end
+
+    def route_loop
+      ready
+      loop { _route }
+    end
+
+    private
+
+    def ready
       @adapter.ready_index
+    end
 
-      @loader.each do |data, file|
-        @adapter.send data
-        SlackScratcher.logger.info "* #{file} is routed."
+    def _route
+      @loader.each(@adapter) do |data, metadata|
+        if data.empty?
+          SlackScratcher.logger.info "* #{metadata} is empty. Nothing happen."
+        else
+          @adapter.send data
+          SlackScratcher.logger.info "* #{metadata} is routed."
+        end
       end
     end
   end

data/lib/slack_scratcher/version.rb

@@ -1,3 +1,3 @@
 module SlackScratcher
-  VERSION = '0.0.1'
+  VERSION = '0.1'
 end

data/lib/slack_scratcher.rb

@@ -1,13 +1,17 @@
 $LOAD_PATH.unshift File.dirname(__FILE__)
 require 'logger'
+require 'dotenv'
+require 'faraday'
+
+Dotenv.load
 
 module SlackScratcher
   autoload :Model, 'slack_scratcher/model'
   autoload :Loader, 'slack_scratcher/loader'
   autoload :Adapter, 'slack_scratcher/adapter'
   autoload :Router, 'slack_scratcher/router'
-  autoload :Collector, 'slack_scratcher/collector'
   autoload :Error, 'slack_scratcher/error'
+  autoload :Helper, 'slack_scratcher/helper'
 
   def self.logger
     @logger ||= ::Logger.new(STDOUT)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: slack_scratcher
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: '0.1'
 platform: ruby
 authors:
 - Daekwon Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-13 00:00:00.000000000 Z
+date: 2015-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
@@ -148,11 +176,12 @@ files:
 - lib/slack_scratcher/adapter.rb
 - lib/slack_scratcher/adapter/elasticsearch.rb
 - lib/slack_scratcher/adapter/file.rb
-- lib/slack_scratcher/collector.rb
 - lib/slack_scratcher/error.rb
 - lib/slack_scratcher/error/file_not_found.rb
 - lib/slack_scratcher/error/slack_api_error.rb
+- lib/slack_scratcher/error/token_not_set.rb
 - lib/slack_scratcher/error/user_not_found_error.rb
+- lib/slack_scratcher/helper.rb
 - lib/slack_scratcher/loader.rb
 - lib/slack_scratcher/loader/api.rb
 - lib/slack_scratcher/loader/file.rb

data/lib/slack_scratcher/collector.rb

@@ -1,16 +0,0 @@
-module SlackScratcher
-  class Collector
-    def initialize(loader, adapter)
-      @loader = loader
-      @adapter = adapter
-    end
-
-    def collect_loop
-      loop do
-        @loader.each(@adapter) do |data|
-          @adapter.send data
-        end
-      end
-    end
-  end
-end