akane 0.0.1

data/lib/akane/recorder.rb

@@ -0,0 +1,104 @@
+require 'thread'
+
+module Akane
+  class Recorder
+    def initialize(storages, logger: Logger.new(nil))
+      @storages = storages
+      @logger = logger
+      @queue = Queue.new
+      @recently_performed = RoundrobinFlags.new(1000)
+    end
+
+    def queue_length
+      @queue.size
+    end
+
+    def record_tweet(account, tweet)
+      @queue << [:record_tweet, account, tweet]
+      self
+    end
+
+    def mark_as_deleted(account, user_id, tweet_id)
+      @queue << [:mark_as_deleted, account, user_id, tweet_id]
+      self
+    end
+
+    def record_message(account, message)
+      @queue << [:record_message, account, message]
+      self
+    end
+
+    def record_event(account, event)
+      @queue << [:record_event, account, event]
+      self
+    end
+
+    def dequeue(raise_errors = false)
+      perform(*@queue.pop, raise_errors: raise_errors)
+    end
+
+    def perform(action, account, *payload, raise_errors: false)
+      if action == :record_tweet
+        return if @recently_performed[payload.last["id"]]
+        @recently_performed.flag!(payload.last["id"])
+
+        if payload.last["retweeted_status"]
+          perform(:record_tweet, account, payload.last["retweeted_status"], raise_errors: raise_errors)
+        end
+      end
+
+      @storages.each do |storage|
+        begin
+          storage.__send__(action, account, *payload)
+        rescue Exception => e
+          raise e if e === Interrupt
+          raise e if raise_errors
+          @logger.error "Error while recorder performing to #{storage.inspect}:  #{e.inspect}"
+          @logger.error e.backtrace
+        end
+      end
+    end
+
+    def run(raise_errors = false)
+      loop do
+        begin
+          self.dequeue(raise_errors)
+        rescue Exception => e
+          raise e if e === Interrupt
+          raise e if raise_errors
+          @logger.error "Error while recorder dequing: #{e.inspect}"
+          @logger.error e.backtrace
+        end
+      end
+    end
+
+    class RoundrobinFlags
+      def initialize(size)
+        @hash = {}
+        @limit_size = size
+      end
+
+      def [](k)
+        @hash[k]
+      end
+
+      def flag!(k)
+        @hash[k] = true
+        if @limit_size < @hash.size
+          @hash.keys.first(@hash.size-@limit_size).each do |overflowed_key|
+            @hash.delete overflowed_key
+          end
+        end
+        nil
+      end
+
+      def unflag!(k)
+        @hash.delete k
+      end
+
+      def flags
+        @hash.keys
+      end
+    end
+  end
+end

data/lib/akane/storages/abstract_storage.rb

@@ -0,0 +1,26 @@
+module Akane
+  module Storages
+    class AbstractStorage
+      def initialize(config: raise(ArgumentError, 'missing config'), logger: Logger.new($stdout))
+        @config = config
+        @logger = logger
+      end
+
+      def record_tweet(account, tweet)
+        raise NotImplementedError
+      end
+
+      def mark_as_deleted(account, user_id, tweet_id)
+        raise NotImplementedError
+      end
+
+      def record_event(account, event)
+        raise NotImplementedError
+      end
+
+      def record_message(account, message)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/akane/storages/elasticsearch.rb

@@ -0,0 +1,242 @@
+require 'akane/storages/abstract_storage'
+require 'elasticsearch'
+
+module Akane
+  module Storages
+    class Elasticsearch < AbstractStorage
+      def initialize(*)
+        super
+
+        @es = ::Elasticsearch::Client.new(
+          hosts: [@config["host"]],
+          logger: @config["enable_es_log"] ? @logger : nil
+        )
+        @index_name = @config["index"] || 'akane'
+        set_elasticsearch_up
+      end
+
+      def record_tweet(account, tweet)
+        tweet_hash = tweet.attrs
+        tweet_hash[:deleted] = false
+        @es.index(index: @index_name, type: 'tweet', id: tweet_hash[:id_str], body: tweet_hash)
+      end
+
+      def mark_as_deleted(account, user_id, tweet_id)
+        tweet = @es.get(index: @index_name, type: 'tweet', id: tweet_id.to_s)['_source']
+        tweet['deleted'] = true
+        @es.index(index: @index_name, type: 'tweet', id: tweet_id.to_s, body: tweet)
+        minimum_tweet = {
+          id: tweet['id'],
+          id_str: tweet['id_str'],
+          text: tweet['text'],
+          user: {
+            id: tweet['user']['id'],
+            id_str: tweet['user']['id_str'],
+            screen_name: tweet['user']['screen_name'],
+          }
+        }
+        @es.index(index: @index_name, type: 'deleted_tweet', id: tweet_id.to_s, body: {tweet: minimum_tweet, deleted_at: Time.now.strftime('%Y-%m-%d %H:%M:%S %z')})
+      rescue ::Elasticsearch::Transport::Transport::Errors::NotFound => e
+        @logger.debug "Due to 404, skipping Deletion for #{tweet_id}"
+        # do nothing
+      end
+
+      def record_event(account, event)
+        case event["event"]
+        when 'favorite'
+        when 'unfavorite'
+        when 'block'
+        when 'unblock'
+        when 'follow'
+        when 'unfollow'
+        end
+      end
+
+      def record_message(account, message)
+        @es.index(index: @index_name, type: 'message', id: message[:id_str], body: message.attrs)
+      end
+
+      private
+
+      def set_elasticsearch_up
+        begin
+          @es.indices.get_mapping(index: @index_name)
+        rescue ::Elasticsearch::Transport::Transport::Errors::NotFound => e
+          raise e unless /IndexMissingException/ === e.message
+
+          @logger.info 'elasticsearch.setup: creating index'
+
+          date_format = "EE MMM d HH:mm:ss Z yyyy"
+          user_properties = {
+                      notifications: {type: 'boolean', store: 'no', index: 'no'},
+                      follow_request_sent: {type: 'boolean', store: 'no', index: 'no'},
+                      following: {type: 'boolean', store: 'no', index: 'no'},
+                      default_profile_image: {type: 'boolean', store: 'no', index: 'no'},
+                      default_profile: {type: 'boolean', store: 'no', index: 'no'},
+                      geo_enabled: {type: 'boolean', store: 'no', index: 'no'},
+                      time_zone: {type: 'string', index: 'not_analyzed'},
+                      utc_offset: {type: 'integer', store: 'yes', index: 'no'},
+                      favourites_count: {type: 'integer', store: 'no', index: 'no'},
+                      created_at: {type: 'date', format: date_format, store: 'yes', index: 'no'},
+                      listed_count: {type: 'integer', store: 'no', index: 'no'},
+                      friends_count: {type: 'integer', store: 'no', index: 'no'},
+                      followers_count: {type: 'integer', store: 'no', index: 'no'},
+                      id: {type: 'long'},
+                      id_str: {type: 'string', index: 'not_analyzed'},
+                      name: {type: 'string'}.merge(
+                        @config["kuromoji"] ? {analyzer: 'kuromoji'} : {}),
+                      screen_name: {type: 'string', index: 'not_analyzed'},
+                      location: {type: 'string', index: 'no'},
+                      url: {type: 'string', index: 'no'},
+                      description: {type: 'string'}.merge(
+                        @config["kuromoji"] ? {analyzer: 'kuromoji'} : {}),
+                      protected: {type: 'boolean'},
+                      verified: {type: 'boolean'},
+                      statuses_count: {type: 'long', store: 'yes', index: 'no'},
+                      lang: {type: 'string', index: 'not_analyzed'},
+                      contributors_enabled: {type: 'boolean', index: 'no'},
+                      is_translator: {type: 'boolean', index: 'no'},
+                      profile_background_color: {type: 'string', store: 'no', index: 'no'},
+                      profile_background_image_url: {type: 'string', store: 'no', index: 'no'},
+                      profile_background_image_url_https: {type: 'string', store: 'no', index: 'no'},
+                      profile_background_tile: {type: 'boolean', store: 'no', index: 'no'},
+                      profile_image_url: {type: 'string', type: 'string', index: 'no'},
+                      profile_image_url_https: {type: 'string', index: 'no'},
+                      profile_link_color: {type: 'string', store: 'no', index: 'no'},
+                      profile_sidebar_border_color: {type: 'string', store: 'no', index: 'no'},
+                      profile_sidebar_fill_color: {type: 'string', store: 'no', index: 'no'},
+                      profile_use_background_image: {type: 'boolean', store: 'no', index: 'no'},
+                    }
+
+          minimum_user_properties = Hash[
+            user_properties.map { |k, v|
+              [k, %i(id id_str screen_name).include?(k) ? v : {type: v[:type], format: v[:format], store: 'no', index: 'no'}] }
+          ]
+
+          tweet_properties = {
+                  lang: {type: 'string', index: 'not_analyzed'},
+                  deleted: {type: 'boolean', null_value: false},
+                  filter_level: {type: 'string', index: 'no'},
+                  retweeted: {type: 'boolean', store: 'no', index: 'no'},
+                  favorited: {type: 'boolean', store: 'no', index: 'no'},
+                  entities: {type: 'boolean', store: 'no', index: 'no'},
+                  favorite_count: {type: 'integer', store: 'no', index: 'no'},
+                  retweet_count: {type: 'integer', store: 'no', index: 'no'},
+                  in_reply_to_status_id_str: {type: 'string', index: 'not_analyzed'},
+                  in_reply_to_status_id: {type: 'long'},
+                  truncated: {type: 'boolean', store: 'no', index: 'no'},
+                  source: {type: 'string'},
+                  text: {type: 'string', boost: 2.0, }.merge(
+                    @config["kuromoji"] ? {analyzer: 'kuromoji'} : {}),
+                  id_str: {type: 'string', index: 'not_analyzed'},
+                  id: {type: 'long'},
+                  created_at: {type: 'date', format: date_format},
+                  in_reply_to_user_id_str: {type: 'string', index: 'not_analyzed'},
+                  in_reply_to_user_id: {type: 'long'},
+                  user: {
+                    type: 'object',
+                    properties: user_properties,
+                  },
+                  coordinates: {
+                    type: 'object',
+                    properties: {
+                      coordinates: {type: 'geo_point'},
+                      type: {type: 'string', index: 'no'},
+                    },
+                  },
+                  place: {
+                    type: 'object',
+                    properties: {
+                      attributes: {type: 'object', store: 'no', index: 'no'},
+                      bounding_box: {type: 'object', index: 'no'},
+                      country: {type: 'string', index: 'no'},
+                      country_code: {type: 'string', index: 'not_analyzed'},
+                      id: {type: 'string', index: 'not_analyzed'},
+                      name: {type: 'string'},
+                      place_type: {type: 'string', index: 'no'},
+                      url: {type: 'string', index: 'no', store: 'yes'},
+                    },
+                  },
+                  contributors: {type: 'object', store: 'no', index: 'no'},
+                }
+
+          minimum_tweet_properties = Hash[
+            tweet_properties.map { |k, v|
+              if k == :user
+                [k, {type: 'object', properties: minimum_user_properties}]
+              else
+                [k, %i(id id_str text).include?(k) ? v : {type: v[:type], format: v[:format], store: 'no', index: 'no'}]
+              end
+            }
+          ]
+          tweet_properties[:retweeted_status] = {type: 'object', properties: minimum_tweet_properties}
+          minimum_tweet_properties[:retweeted_status] = {type: 'object', store: 'no', index: 'no'}
+
+          @es.indices.create(index: @index_name, body: {
+            settings: {
+            },
+            analysis: {
+              standard: {
+                  type: 'standard'
+              },
+            }.merge( @config["kuromoji"] ?
+                {kuromoji: {
+                  type: "kuromoji_tokenizer",
+                  mode: "search",
+                }} : {}
+              ),
+            mappings: {
+              tweet: {
+                _source: {enabled: true},
+                properties: tweet_properties,
+              },
+              deleted_tweet: {
+                _source: {enabled: true},
+                properties: {
+                  tweet: {type: 'object', properties: minimum_tweet_properties},
+                  deleted_at: {type: 'date', index: 'no'},
+                },
+              },
+
+              message: {
+                _source: {enabled: true},
+                properties: {
+                  created_at: {type: 'date', format: date_format, store: 'yes', index: 'no'},
+                  text: {type: 'string', boost: 2.0, store: 'yes', }.merge(
+                    @config["kuromoji"] ? {analyzer: 'kuromoji'} : {}),
+                  sender_id_str: {type: 'string', store: 'yes', index: 'not_analyzed'},
+                  sender_screen_name: {type: 'string', store: 'yes', index: 'not_analyzed'},
+                  sender_id: {type: 'long', store: 'yes', },
+                  recipient_id_str: {type: 'string', store: 'yes', index: 'not_analyzed'},
+                  recipient_id: {type: 'long', store: 'yes', },
+                  recipient_screen_name: {type: 'string', store: 'yes', index: 'not_analyzed'},
+                  sender: {type: 'object', store: 'yes', properties: minimum_user_properties},
+                  recipient: {type: 'object', store: 'yes', properties: minimum_user_properties},
+                },
+              },
+              event_favorite: {
+                _source: {enabled: true},
+                properties: {
+                  created_at: {type: 'date', format: date_format, store: 'yes', index: 'no'},
+                  event: {type: 'string', store: 'yes', index: 'not_analyzed'},
+                  source: {type: 'object', store: 'yes', properties: minimum_user_properties},
+                  target: {type: 'object', store: 'yes', properties: minimum_user_properties},
+                  target_object: {type: 'object', store: 'yes', properties: minimum_tweet_properties},
+                },
+              },
+              event_user_interaction: {
+                _source: {enabled: true},
+                properties: {
+                  created_at: {type: 'date', format: date_format, store: 'yes', index: 'no'},
+                  event: {type: 'string', store: 'yes', index: 'not_analyzed'},
+                  source: {type: 'object', store: 'yes', properties: minimum_user_properties},
+                  target: {type: 'object', store: 'yes', properties: minimum_user_properties},
+                },
+              },
+            },
+          })
+        end
+      end
+    end
+  end
+end

data/lib/akane/storages/file.rb

@@ -0,0 +1,142 @@
+require 'akane/storages/abstract_storage'
+require 'date'
+require 'json'
+require 'time'
+require 'pathname'
+
+module Akane
+  module Storages
+    class File < AbstractStorage
+      def initialize(*)
+        super
+        @screen_name_to_id_cache = {}
+        @dir = Pathname.new(@config["dir"])
+        [@dir, @dir.join('names'), @dir.join('users'), @dir.join('event'), @dir.join('timeline')].each do |d|
+          d.mkdir unless d.exist?
+        end
+      end
+
+      def record_tweet(account, tweet)
+        timeline_io.puts "[#{tweet["created_at"].xmlschema}][#{account}] #{tweet["user"]["screen_name"]}: " \
+                         "#{tweet["text"].gsub(/\r?\n/,' ')} (#{tweet["user"]["id"]},#{tweet["id"]})"
+
+        tweets_io_for_user(tweet["user"]["id"], tweet["user"]["screen_name"]) do |io|
+          io.puts tweet.attrs.to_json
+        end
+      end
+
+      def mark_as_deleted(account, user_id, tweet_id)
+        timeline_deletion_io.puts "#{Time.now.xmlschema},#{user_id},#{tweet_id}"
+        tweets_deletion_io_for_user(user_id) do |io|
+          io.puts "#{Time.now.xmlschema},#{user_id},#{tweet_id}"
+        end
+      end
+
+      def record_event(account, event)
+        event_io.puts event.merge("happened_on" => account).to_json
+      end
+
+      def record_message(account, message)
+        messages_raw_io_for_user(message["sender"]["id"], message["sender"]["screen_name"]) do |io|
+          io.puts message.attrs.to_json
+        end
+        messages_io_for_user(message["sender"]["id"], message["sender"]["screen_name"]) do |io|
+          io.puts "[#{message["created_at"].xmlschema}] #{message["sender"]["screen_name"]} -> #{message["recipient"]["screen_name"]}:" \
+                           " #{message["text"]} (#{message["sender"]["id"]} -> #{message["recipient"]["id"]},#{message["id"]})"
+        end
+      end
+
+      private
+
+      def timeline_io
+        if @timeline_io_date != Date.today || !@timeline_io
+          date = @timeline_io_date = Date.today
+          @timeline_io = ::File.open(@dir.join('timeline', date.strftime('%Y-%m-%d.txt')), 'a')
+          @timeline_io.sync = !@config.key?("sync_io") || @config["sync_io"]
+          @timeline_io
+        else
+          @timeline_io
+        end
+      end
+
+      def timeline_deletion_io
+        if @timeline_deletion_io_date != Date.today || !@timeline_deletion_io
+          date = @timeline_deletion_io_date = Date.today
+          @timeline_deletion_io = ::File.open(@dir.join('timeline', date.strftime('%Y-%m-%d.deleted.txt')), 'a')
+          @timeline_deletion_io.sync = !@config.key?("sync_io") || @config["sync_io"]
+          @timeline_deletion_io
+        else
+          @timeline_deletion_io
+        end
+      end
+
+      def event_io
+        if @event_io_date != Date.today || !@event_io
+          date = @event_io_date = Date.today
+          @event_io = ::File.open(@dir.join('event', date.strftime('%Y-%m-%d.txt')), 'a')
+          @event_io.sync = !@config.key?("sync_io") || @config["sync_io"]
+          @event_io
+        else
+          @event_io
+        end
+      end
+
+      def tweets_io_for_user(user_id, screen_name=nil, &block)
+        symlink_user_dir(user_id, screen_name)
+        date = Date.today
+        ::File.open(@dir.join('users', user_id.to_s, date.strftime('tweets.%Y-%m.txt')), 'a', &block)
+      end
+
+      def tweets_deletion_io_for_user(user_id, screen_name=nil, &block)
+        symlink_user_dir(user_id, screen_name)
+        date = Date.today
+        ::File.open(@dir.join('users', user_id.to_s, date.strftime('deleted-tweets.%Y-%m.txt')), 'a', &block)
+      end
+
+      def messages_io_for_user(user_id, screen_name=nil, &block)
+        symlink_user_dir(user_id, screen_name)
+        date = Date.today
+        ::File.open(@dir.join('users', user_id.to_s, date.strftime('messages.%Y-%m.txt')), 'a', &block)
+      end
+
+      def messages_raw_io_for_user(user_id, screen_name=nil, &block)
+        symlink_user_dir(user_id, screen_name)
+        date = Date.today
+        ::File.open(@dir.join('users', user_id.to_s, date.strftime('messages-raw.%Y-%m.txt')), 'a', &block)
+      end
+
+
+      def symlink_user_dir(user_id, screen_name=nil)
+        user_id_dir = @dir.join('users', user_id.to_s)
+        user_id_dir.mkdir unless user_id_dir.exist?
+
+        return unless screen_name
+        screen_name_dir = @dir.join('names', screen_name)
+
+        unless @screen_name_to_id_cache.has_key?(screen_name)
+          @screen_name_to_id_cache[screen_name] = if screen_name_dir.symlink?
+                                                    screen_name_dir.realpath.basename.to_s
+                                                  else
+                                                    nil
+                                                  end
+          @logger.debug "Caching dir for #{screen_name} : #{@screen_name_to_id_cache[screen_name].inspect}"
+        end
+
+        cached_id = @screen_name_to_id_cache[screen_name]
+
+        if cached_id && cached_id != user_id.to_s
+          prev_id = screen_name_dir.realpath.basename
+          @logger.info "Renaming #{screen_name}(#{prev_id}) dir: #{screen_name} -> #{prev_id}-#{screen_name}"
+          screen_name_dir.rename(@dir.join('names',"#{prev_id}-#{screen_name}"))
+          screen_name_dir.make_symlink("../users/#{user_id_dir.basename}")
+          @screen_name_to_id_cache[screen_name] = user_id.to_s
+
+        elsif cached_id.nil?
+          @logger.info "Linking #{screen_name}->#{user_id} dir"
+          screen_name_dir.make_symlink("../users/#{user_id_dir.basename}")
+          @screen_name_to_id_cache[screen_name] = user_id.to_s
+        end
+      end
+    end
+  end
+end