empyrean 0.1.0

data/README.md

@@ -0,0 +1,25 @@
+# Empyrean
+
+Generate full stats of your account using your Twitter Archive.
+
+## Usage
+
+1. Install Empyrean using `gem install empyrean`
+1. Download your Twitter archive and unpack it somewhere.
+2. Generate a new configuration file: `empyrean -g config.yml`
+2. `empyrean -d path/to/data/js/tweets`
+3. ???
+4. Profit!
+
+## Configuration
+
+Configuration is done using the `config.yml` file in the current directory.
+You can generate one using `empyrean -g config.yml`.  You may also specify a
+different configuration file to use with the `-c` switch.
+
+You can also use the `-C` switch to override config values on the fly, such as
+`-C timezone_difference=2` or `-C mentions_enabled=no`.  If the configuration
+variable is a list, separate the elements with `,`.  For example, to ignore
+the Twitter users `BarackObama` and `baddragon_en`, use
+`-C ignored_users=BarackObama,baddragon_en`.
+

data/bin/empyrean

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# empyrean - generate a statistics page from your Twitter archive
+#
+# This file is part of Empyrean
+# Copyright (C) 2015 nilsding, pixeldesu
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'empyrean/cli'
+
+Empyrean::CLI.new(*ARGV)

data/config.yml.example

@@ -0,0 +1,31 @@
+# Empyrean config file
+
+timezone_difference: 0  # Time zone difference (e.g. 2 for UTC+2, -6 for UTC-6)
+
+mentions:
+  enabled: true   # Enable "Most mentioned users" list
+  top: 10         # How many users to show on the "Most mentioned users" list
+  notop: 20       # How many users to show on the "These didn't make it to the top" list
+clients:
+  enabled: true   # Enable "Most used clients" list
+  top: 10         # How many clients to show on the "Most used clients" list
+  notop: 20       # How many clients to show on the "These didn't make it to the top" list
+hashtags:
+  enabled: true   # Enable "Most used hashtags" list
+  top: 10         # How many hashtags to show on the "Most used hashtags" list
+  notop: 20       # How many hashtags to show on the "These didn't make it to the top" list
+smileys:
+  enabled: true   # Enable "Most used smileys" list
+  top: 10         # How many smileys to show on the "Most used smileys" list
+  notop: 0        # How many smileys to show on the "These didn't make it to the top" list
+
+ignored_users:    # Users to exclude from the statistics.
+  - BarackObama
+  - tbhjuststop
+
+renamed_users:    # Users that have renamed themselves
+  pixeldesu:     "mochizune" # mochizune was previously known as pixeldesu
+  technikaflash: "mochizune" # mochizune was also previously known as TechnikaFlash (the downcase on the username is important)
+  traxxed22:     "mochizune" # and before that, mochizune was known as TraXXed22.
+
+# kate: indent-width 2

data/empyrean.gemspec

@@ -0,0 +1,20 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'empyrean/defaults'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'empyrean'
+  spec.version       = Empyrean::VERSION
+  spec.authors       = ['nilsding', 'pixeldesu']
+  spec.email         = ['nilsding@nilsding.org', 'andy@pixelde.su']
+  spec.summary       = %q{Generates stats using your Twitter archive.}
+  spec.description   = %q{With Empyrean, you can generate full stats of your Twitter account using your Twitter archive.}
+  spec.homepage      = 'https://github.com/Leafcat/Empyrean'
+  spec.license       = 'GPLv3'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+end

data/lib/empyrean/cli.rb

@@ -0,0 +1,97 @@
+# cli.rb
+#
+# This file is part of Empyrean
+# Copyright (C) 2015 nilsding, pixeldesu
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'empyrean/defaults'
+require 'empyrean/optparser'
+require 'empyrean/configloader'
+require 'empyrean/tweetloader'
+require 'empyrean/tweetparser'
+require 'empyrean/templatelister'
+require 'empyrean/templaterenderer'
+
+module Empyrean
+  class CLI
+    def initialize(*argv)
+      @options = OptParser.parse(argv)
+      @config = ConfigLoader.new(@options).load_config
+      
+      parsed = analyze_tweets
+      print_stats(parsed)
+      generate_html(parsed)
+    end
+    
+    def analyze_tweets
+      tweet_files = TweetLoader.read_directory(@options.jsondir)
+      parsed = []
+      parser = TweetParser.new(@options, @config)
+      tweet_files.each do |file|
+        tweet = TweetLoader.read_file file
+        parsed << parser.parse(tweet)
+      end
+      TweetParser.merge_parsed parsed
+    end
+    
+    def print_stats(parsed)
+      puts "Analyzed #{parsed[:tweet_count]} tweets"
+      puts "  - #{(parsed[:retweet_count] * 100 / parsed[:tweet_count].to_f).round(2)}% retweets\n"
+      return unless @options.print_stats
+
+      headers = {
+        mentions: ["You send most mentions to:", "times",  :name],
+        clients:  ["Most used clients:",         "tweets", :name],
+        hashtags: ["Your most used hashtags:",   "times",  :hashtag],
+        smileys:  ["Your most used smileys:",    "times",  :smiley]
+      }
+      
+      headers.each do |k, v|
+        if @config[k][:enabled]
+          puts v[0]
+          begin
+            @config[k][:top].times do |i|
+              puts "#{sprintf "%2d", i + 1}. #{'#' if k == :hashtags}#{parsed[k][i][1][v[2]]} (#{parsed[k][i][1][:count]} #{v[1]})"
+            end
+          rescue => _
+          end
+        end
+      end
+    end
+    
+    def generate_html(parsed)
+      puts "Generating HTML"
+      template = File.read template_file
+      renderer = TemplateRenderer.new @config, template, parsed
+      File.open(@options.outfile, 'w') do |outfile|
+        outfile.write renderer.render
+        outfile.flush
+      end
+      puts " => #{@options.outfile}"
+    end
+    
+    def template_file
+      if File.exist? File.join(Dir.pwd, @options.template)
+        File.join(Dir.pwd, @options.template)
+      elsif TemplateLister.list.include? @options.template
+        File.join TEMPLATE_DIR, @options.template
+      else
+        @options.template
+      end
+    end
+  end
+end
+
+# kate: indent-width 2

data/lib/empyrean/configloader.rb

@@ -0,0 +1,105 @@
+# configloader.rb - loads a config file
+#
+# This file is part of Empyrean
+# Copyright (C) 2015 nilsding, pixeldesu
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'yaml'
+require 'empyrean/defaults'
+
+module Empyrean
+  class ConfigLoader
+    def initialize(options)
+      @options = options
+    end
+    
+    ##
+    # Loads a YAML file, parses it and returns a hash with symbolized keys.
+    def load(file)
+      if File.exist? file
+        symbolize_keys(YAML.load_file(File.expand_path('.', file)))
+      else
+        {}
+      end
+    end
+
+    # Loads a YAML file, parses it and checks if all values are given.  If a
+    # value is missing, it will be set with the default value.
+    def load_config(file = @options.config)
+      config = load(file)
+      config[:timezone_difference] = 0        if config[:timezone_difference].nil?
+      config[:mentions]            = {}       if config[:mentions].nil?
+      config[:mentions][:enabled]  = true     if config[:mentions][:enabled].nil?
+      config[:mentions][:top]      = 10       if config[:mentions][:top].nil?
+      config[:mentions][:notop]    = 20       if config[:mentions][:notop].nil?
+      config[:clients]             = {}       if config[:clients].nil?
+      config[:clients][:enabled]   = true     if config[:clients][:enabled].nil?
+      config[:clients][:top]       = 10       if config[:clients][:top].nil?
+      config[:clients][:notop]     = 20       if config[:clients][:notop].nil?
+      config[:hashtags]            = {}       if config[:hashtags].nil?
+      config[:hashtags][:enabled]  = true     if config[:hashtags][:enabled].nil?
+      config[:hashtags][:top]      = 10       if config[:hashtags][:top].nil?
+      config[:hashtags][:notop]    = 20       if config[:hashtags][:notop].nil?
+      config[:smileys]             = {}       if config[:smileys].nil?
+      config[:smileys][:enabled]   = true     if config[:smileys][:enabled].nil?
+      config[:smileys][:top]       = 10       if config[:smileys][:top].nil?
+      config[:smileys][:notop]     = 0        if config[:smileys][:notop].nil?
+      config[:ignored_users]       = []       if config[:ignored_users].nil?
+      config[:renamed_users]       = []       if config[:renamed_users].nil?
+      args_override config
+    end
+
+    def args_override(config)
+      config_args = @options.config_values
+      config[:timezone_difference] = config_args[:timezone_difference] unless config_args[:timezone_difference].nil?
+      config[:mentions][:enabled]  = config_args[:mentions_enabled]    unless config_args[:mentions_enabled].nil?
+      config[:mentions][:top]      = config_args[:mentions_top]        unless config_args[:mentions_top].nil?
+      config[:mentions][:notop]    = config_args[:mentions_notop]      unless config_args[:mentions_notop].nil?
+      config[:clients][:enabled]   = config_args[:clients_enabled]     unless config_args[:clients_enabled].nil?
+      config[:clients][:top]       = config_args[:clients_top]         unless config_args[:clients_top].nil?
+      config[:clients][:notop]     = config_args[:clients_notop]       unless config_args[:clients_notop].nil?
+      config[:hashtags][:enabled]  = config_args[:hashtags_enabled]    unless config_args[:hashtags_enabled].nil?
+      config[:hashtags][:top]      = config_args[:hashtags_top]        unless config_args[:hashtags_top].nil?
+      config[:hashtags][:notop]    = config_args[:hashtags_notop]      unless config_args[:hashtags_notop].nil?
+      config[:smileys][:enabled]   = config_args[:smileys_enabled]     unless config_args[:smileys_enabled].nil?
+      config[:smileys][:top]       = config_args[:smileys_top]         unless config_args[:smileys_top].nil?
+      config[:smileys][:notop]     = config_args[:smileys_notop]       unless config_args[:smileys_notop].nil?
+      config[:ignored_users]       = config_args[:ignored_users]       unless config_args[:ignored_users].nil?
+      config[:ignored_users].each do |user|    user.downcase! end
+      config[:renamed_users].each do |old, new| new.downcase! end
+      config
+    end
+
+    private
+    
+    # Symbolizes the keys of a hash, duh.
+    def symbolize_keys(hash)
+      hash.inject({}) do |result, (key, value)|
+        new_key = case key
+                  when String then key.to_sym
+                  else key
+                  end
+        new_value = case value
+                    when Hash then symbolize_keys(value)
+                    else value
+                    end
+        result[new_key] = new_value
+        result
+      end
+    end
+  end
+end
+
+# kate: indent-width 2

data/lib/empyrean/defaults.rb

@@ -0,0 +1,43 @@
+# defaults.rb - some constants
+#
+# This file is part of Empyrean
+# Copyright (C) 2015 nilsding, pixeldesu
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+module Empyrean
+  # Application name
+  APP_NAME = "Empyrean"
+
+  # Version
+  VERSION = "0.1.0"
+
+  # Combined version string
+  VERSION_STR = "#{APP_NAME} #{VERSION}"
+
+  # Regexp for matching user names
+  USERNAME_REGEX = /[@]([a-zA-Z0-9_]{1,16})/
+
+  # Regexp for matching the client source
+  SOURCE_REGEX = /^<a href=\"(https?:\/\/\S+|erased_\d+)\" rel=\"nofollow\">(.+)<\/a>$/
+
+  # Regexp for matching hashtags
+  HASHTAG_REGEX = /[#]([^{}\[\]().,\-:!*_?#\s]+)/
+  
+  # Path to the templates
+  TEMPLATE_DIR = File.expand_path "../templates", __FILE__
+  
+  # The default template to use
+  DEFAULT_TEMPLATE = "default.html.erb"
+end

data/lib/empyrean/optparser.rb

@@ -0,0 +1,151 @@
+# optparser.rb - parses CLI options using Ruby's OptionParser
+#
+# This file is part of Empyrean
+# Copyright (C) 2015 nilsding, pixeldesu
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'optparse'
+require 'ostruct'
+require 'empyrean/defaults'
+require 'empyrean/templatelister'
+
+module Empyrean
+  class OptParser
+
+    def self.parse(args)
+      options = OpenStruct.new
+      options.jsondir = ""
+      options.outfile = "output.html"
+      options.config = File.expand_path('.', "config.yml")
+      options.config_values = {}
+      options.print_stats = false
+      options.template = DEFAULT_TEMPLATE
+      options.verbose = false
+
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: stats.rb [options]"
+
+        opts.separator ""
+        opts.separator "Specific options:"
+        
+        opts.on("-c", "--config CONFIG", "The configuration file to use (default: #{options.config})") do |config|
+          options.config = config
+        end
+
+        opts.on("-C", "--config-value KEY=VALUE", "Sets a configuration value (e.g. hashtags_enabled=false)") do |val|
+          key, value = val.split("=")
+          key = key.downcase
+          unless value.nil?   # ignore empty values
+            case key.to_s
+            when /_enabled$/
+              value = to_bool value
+            when /_(no)?top$/, /timezone_difference$/
+              value = value.to_i
+            when /ignored_users/
+              value = value.split ','
+            end
+            options.config_values[key.to_sym] = value
+          end
+        end
+
+        opts.on("-g", "--generate-config CONFIG", "Generate a new configuration file") do |config_file|
+          fn = File.expand_path '.', config_file
+          if File.exist? fn
+            STDERR.puts "Cowardly refusing to overwrite already existing file #{config_file}."
+            exit 2
+          end
+          File.open fn, 'w' do |f|
+            f.write File.read File.expand_path("../../../config.yml.example", __FILE__)
+            STDERR.puts "Wrote new configuration to #{config_file}."
+            exit
+          end
+        end
+
+        opts.on("-d", "--jsondir DIRECTORY", "Directory with tweet files (containing 2014_07.js etc.)") do |dir|
+          dir = File.expand_path('.', dir)
+          unless File.exist?(dir) && File.directory?(dir)
+            STDERR.puts "not a directory: #{dir}"
+            exit 1
+          end
+
+          entries = Dir.entries(dir)
+          entries.each do |e|
+            # the file names of the tweet archive are in the format /^\d{4}_\d{2}\.js$/
+            unless e =~ /^\d{4}_\d{2}\.js$|\.+?/
+              STDERR.puts "not a tweets directory: #{dir}"
+              exit 1
+            end
+          end
+
+          options.jsondir = dir
+        end
+
+        opts.on("-o", "--outfile OUTFILE", "Output HTML file (default: #{options.outfile})") do |outfile|
+          options.outfile = outfile
+        end
+        
+        opts.on("-l", "--list-templates", "List available templates") do
+          TemplateLister.print_list
+          exit
+        end
+
+        opts.on("-t", "--template TEMPLATE", "Template to use (default: #{options.template})") do |template|
+          options.template = template
+        end
+
+        opts.on("-p", "--[no-]print-stats", "Print stats to stdout") do |p|
+          options.print_stats = p
+        end
+
+        opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+          options.verbose = v
+        end
+        
+        # No argument, shows at tail.  This will print an options summary.
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        # Another typical switch to print the version.
+        opts.on_tail("--version", "Show version") do
+          puts Empyrean::VERSION_STR
+          exit
+        end
+      end.parse!(args)
+
+      if options.jsondir.empty?
+        STDERR.puts "missing argument: --jsondir, see --help for more"
+        exit 1
+      end
+
+      options
+    end # parse()
+
+    private
+
+    ##
+    # "converts" a string to a boolean value
+    def self.to_bool str
+      if str == true || str =~ (/(true|t|yes|y|1)$/i)
+        true
+      else
+        false
+      end
+    end
+  end
+end
+
+# kate: indent-width 2

data/lib/empyrean/templatelister.rb

@@ -0,0 +1,35 @@
+# templatelister.rb - lists available templates
+#
+# This file is part of Empyrean
+# Copyright (C) 2015 nilsding, pixeldesu
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'empyrean/defaults'
+
+module Empyrean
+  class TemplateLister
+    class << self
+      # Returns an array of available templates.
+      def list
+        Dir[File.join TEMPLATE_DIR, "*.html.erb"].map{ |t| File.basename t }
+      end
+      
+      # Prints the available templates to stdout.
+      def print_list
+        puts list
+      end
+    end
+  end
+end