twords 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff2afbb65117a20ea8ae1fae8889372a292fbe50
-  data.tar.gz: 1315cb7fe01bb2d995337929df84934b3c22e167
+  metadata.gz: deb67eadb91095b9a3ef2a0cefe7c9452038c0f1
+  data.tar.gz: 7cb612c4aa544038bfb9eb2c88c5367269b1c4c6
 SHA512:
-  metadata.gz: 0b26b32f09d1b70c235da6c0fd8af11660b25412d39489e84022caed28d5ec64033e5f78adf9ffb617182d52322161fc7fab54e873366346c8a04af3544481b9
-  data.tar.gz: 5d4161ecb3543542b1622817e9079e373d5a0222db3c12f35afbef2332876887a4ad86c858856b606d7f9450ac4cba333c8f1a9972cb0d215e4ca0126427830d
+  metadata.gz: 615f31dd12aea64aa8ff1304e947f36706a2e23145ca35153227aac72999359d87bfa2ef97c5f6beaae258be2f97f403c1cba7c78502e7c28991318da081768c
+  data.tar.gz: 49357b0e2060a1726956fb5c1f996ff2281eb2306a4f2ebe1fa2176851d95af5181c5c4c2f5610cd36db782193f6c60776a374d7a7ac70d8f73b79b3a3802394

data/README.md

@@ -22,35 +22,59 @@ Or install it yourself as:
 
 ## Usage
 
+Twords takes a configuration block, and if it doesn't find one it will set the following defaults:
+
 ```ruby
 Twords.config do |config|
-  config.rejects = %w[are this is from be on the for to and at our of in rt a with &] # these words will not be counted (example, not a default; you must provide your own list)
-  config.range   = 14 # number of days to check, no default
-  
-  config.include_hashtags = false # default, excludes words beginning with '#'
-  config.include_uris     = false # default, uses URI#regexp to match
-  config.include_mentions = false # default, excludes words beginning with '@'
-  
+  config.rejects = %w[my us we an w/ because b/c or are this is from
+                      be on the for to and at our of in rt a with &
+                      that it by as if was] # These words will not be counted
+
+  config.range   = 30 # Number of days to check
+
+  config.include_hashtags = false # Excludes strings beginning with '#'
+  config.include_uris     = false # Excludes strings that match URI#regexp
+  config.include_mentions = false # Excludes strings beginning with '@'
+
   config.up_to { Time.now } # The block must return an object that responds to #to_time. The time is lazy evaluated and the range is counted backward from here.
-  
+
+  # By default the Twitter client will look for keys stored as system variables by the names listed below. Feel free to change the configuration, but never hard code the keys.
   config.twitter_client do |twitter|
-    twitter.consumer_key        = YOUR_TWITTER_CONSUMER_KEY
-    twitter.consumer_secret     = YOUR_TWITTER_CONSUMER_SECRET
-    twitter.access_token        = YOUR_TWITTER_ACCESS_TOKEN
-    twitter.access_token_secret = YOUR_TWITTER_ACCESS_TOKEN_SECRET
+    twitter.consumer_key        = ENV['TWITTER_CONSUMER_KEY']
+    twitter.consumer_secret     = ENV['TWITTER_CONSUMER_SECRET']
+    twitter.access_token        = ENV['TWITTER_ACCESS_TOKEN']
+    twitter.access_token_secret = ENV['TWITTER_ACCESS_TOKEN_SECRET']
   end
 end
 
 twords = Twords.new 'user_one', 'user_two' # A list of Twitter handles to include in the count.
 
 twords.audit
+# Fetched user_one's timeline
+# Fetched user_two's timeline
 # => true
 
 twords.words
-# => { "butts"=>32, "poo"=>28, "pups"=>36, ... }
+# => { "pizza"=>32, "burger"=>28, "pups"=>36, ... }
 
 twords.words_forward # Sort descending. Alias #sort_words
-# => [["pups", 36], ["butts", 32], ["poo", 28], ...]
+# => [["pups", 36], ["pizza", 32], ["burger", 28], ...]
+
+Twords.config { |config| config.include_hashtags = true }
+
+twords.audit
+# => true
+
+twords.words
+# => { "pizza"=>32, "burger"=>28, "pups"=>36, ... }
+
+twords.audit!
+# Fetched user_one's timeline
+# Fetched user_two's timeline
+# => true
+
+twords.words
+# => { "#TACOSTACOSTACOS"=>14321, "pizza"=>32, "burger"=>28, "pups"=>36, ... }
 ```
 
 ## Development
@@ -67,4 +91,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/msimon
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/lib/twords.rb

@@ -1,15 +1,19 @@
 # frozen_string_literal: true
 
-require 'csv'
-require 'twitter'
-require 'uri'
-
+require 'twords/configuration'
+require 'twords/instance_methods'
 require 'twords/version'
 
 # Twords.config do |config|
-#   config.rejects = %w[the for and a i of if]
-#   config.max_age     = 14
+#   config.rejects = %w[my us we an w/ because b/c or are this is from
+#                       be on the for to and at our of in rt a with &
+#                       that it by as if was]
+
+#   config.range   = 30
 #   config.up_to { Time.now }
+#   config.include_hashtags = false
+#   config.include_uris     = false
+#   config.include_mentions = false
 #
 #   config.twitter_client do |twitter|
 #     twitter.consumer_key        = YOUR_TWITTER_CONSUMER_KEY
@@ -25,212 +29,18 @@ require 'twords/version'
 # # => true
 #
 # twords.words
-# # => { "butts"=>35, "poo"=>32, "pups"=>28, ... }
+# # => { "pizza"=>32, "burger"=>28, "pups"=>36, ... }
 class Twords
-  class << self
-    attr_reader :rejects, :client, :up_to_block, :include_hashtags, :include_uris,
-                :include_mentions
-    attr_accessor :range
-
-    def config
-      yield self
-    end
-
-    def twitter_client(&block)
-      @client = Twitter::REST::Client.new(&block)
-    end
-
-    def rejects=(*args)
-      @rejects = args.flatten
-    end
-
-    def include_hashtags=(boolean)
-      not_a_boolean_error(boolean)
-      @include_hashtags = boolean
-    end
-
-    def include_uris=(boolean)
-      not_a_boolean_error(boolean)
-      @include_uris = boolean
-    end
-    alias include_urls include_uris
-
-    def include_mentions=(boolean)
-      not_a_boolean_error(boolean)
-      @include_mentions = boolean
-    end
-
-    def not_a_boolean_error(boolean)
-      raise ArgumentError, 'argument must be a booolean value' unless a_boolean?(boolean)
-    end
-
-    def a_boolean?(other)
-      [true, false].include?(other)
-    end
-
-    def up_to(&time_block)
-      @up_to_block = time_block
-    end
-  end
-
-  attr_reader :screen_names, :words, :requests, :client
-
-  def initialize(*screen_names)
-    @screen_names = screen_names.flatten
-    @words        = {}
-    @requests     = 0
-  end
-
-  def client
-    @_client ||= self.class.client
-  end
-
-  def range
-    @_range ||= self.class.range
-  end
-
-  def rejects
-    @_rejects ||= self.class.rejects
-  end
-
-  def audited?
-    @audited
-  end
-
-  def hashtag?(word)
-    return false if self.class.include_hashtags
-    !(word =~ /#(\w+)/).nil?
-  end
-
-  def uri?(word)
-    return false if self.class.include_uris
-    !(word =~ URI.regexp).nil?
-  end
-
-  def mention?(word)
-    return false if self.class.include_mentions
-    !(word =~ /@(\w+)/).nil?
-  end
-
-  def hashtags
-    /#/
-  end
-
-  def should_be_skipped?(word)
-    rejects.include?(word) || hashtag?(word) || uri?(word) || mention?(word)
-  end
-
-  def sort_words
-    words.sort { |a, b| b.last <=> a.last }
-  end
-  alias words_forward sort_words
-
-  def timeline
-    @_timeline ||= screen_names.map { |name| fetch_timeline(name) }.flatten
-  end
-
-  # Make two cursored API calls to fetch the 400 most recent tweets
-  def fetch_timeline(screen_name)
-    return [] if screen_name.to_s.empty?
-    @requests += 1
-    timeline = client.user_timeline(screen_name, tweet_mode: 'extended', count: 200)
-    return timeline if timeline.empty?
-    timeline = fetch_older_tweets(timeline, screen_name)
-    puts "Fetched #{screen_name}'s timeline"
-    timeline
-  end
-
-  def fetch_older_tweets(timeline, screen_name)
-    return timeline if age_of_tweet_in_days(timeline.last) > range
-    @requests += 1
-    first_count = timeline.count
-    timeline += client.user_timeline(
-      screen_name,
-      tweet_mode: 'extended',
-      max_id: timeline.last.id - 1,
-      count: 200
-    )
-    second_count = timeline.count
-    return timeline if second_count == first_count
-    fetch_older_tweets(timeline, screen_name)
-  end
-
-  def tweets
-    @_tweets ||= timeline.each_with_object([]) do |tweet, memo|
-      next if tweet.created_at > up_to_time
-      memo << tweet if age_of_tweet_in_days(tweet) <= range
-    end
-  end
-
-  def sort_tweets
-    tweets.sort { |a, b| b.created_at <=> a.created_at }
-  end
-
-  def sort_tweets!
-    tweets.sort! { |a, b| b.created_at <=> a.created_at }
-  end
-
-  def age_of_tweet_in_days(tweet)
-    (up_to_time - tweet.created_at) / 86_400
-  end
-
-  def up_to_time
-    self.class.up_to_block.call.to_time
-  end
-
-  def count_words
-    words.clear
-    tweets.each do |tweet|
-      words_array = tweet.attrs[:full_text].downcase.split(' ')
-      words_array.each do |word|
-        next if should_be_skipped?(word)
-        if words.key?(word)
-          words[word] += 1
-        else
-          words[word] = 1
-        end
-      end
-    end
-  end
-
-  def audit
-    count_words unless audited?
-    @audited = true
-  end
-
-  def audit!
-    @audited = false
-    audit
-  end
-
-  def tweets_count
-    @_tweets_count ||= tweets.count
-  end
-
-  def to_csv
-    CSV.generate do |csv|
-      csv << %w[word count]
-      sort_words.each do |word_count|
-        csv << word_count
-      end
-    end
-  end
-
-  def write_to_csv(opts = {})
-    filename = opts.fetch(:filename) { 'twords_report.csv' }
-    write_file(filename, :to_csv, opts)
-  end
-
-  def to_json
-    sort_words.to_h.to_json
+  def self.config
+    @configuration ||= Configuration.new
+    @configuration.tap { |config| yield config if block_given? }
   end
 
-  def write_to_json(opts = {})
-    filename = opts.fetch(:filename) { 'twords_report.json' }
-    write_file(filename, :to_json, opts)
+  def self.reset_config!
+    config.reset!
   end
 
-  def write_file(filename, method, opts = {})
-    File.open(filename, 'w', opts) { |file| file.write send(method) }
+  def self.client
+    config.client
   end
 end

data/lib/twords/config_accessible.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class Twords
+  # include ConfigAccessable to access shared configuration settings
+  module ConfigAccessible
+    module_function
+
+    def config
+      Twords.config
+    end
+  end
+end

data/lib/twords/configuration.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'twitter'
+require 'twords/twitter_client'
+
+class Twords
+  # Configuration object
+  class Configuration
+    DEFAULT_REJECTS = %w[
+      my us we an w/ because
+      b/c or are this is from
+      be on the for to and at
+      our of in rt a with &
+      that it by as if was
+    ].freeze
+
+    DEFAULT_TWITTER_CONFIG = lambda do |twitter|
+      twitter.consumer_key        = ENV['TWITTER_CONSUMER_KEY']
+      twitter.consumer_secret     = ENV['TWITTER_CONSUMER_SECRET']
+      twitter.access_token        = ENV['TWITTER_ACCESS_TOKEN']
+      twitter.access_token_secret = ENV['TWITTER_ACCESS_TOKEN_SECRET']
+    end
+
+    DEFAULT_OPTIONS = {
+      include_uris:     false,
+      include_hashtags: false,
+      include_mentions: false,
+      range:            30,
+      client:           TwitterClient.new(&DEFAULT_TWITTER_CONFIG),
+      up_to_block:      -> { Time.now },
+      rejects:          DEFAULT_REJECTS
+    }.freeze
+
+    attr_reader :rejects, :client, :up_to_block, :include_hashtags, :include_uris,
+                :include_mentions
+
+    attr_accessor :range
+
+    def initialize
+      set_defaults
+    end
+
+    def reset!
+      tap { set_defaults }
+    end
+
+    def twitter_client(&block)
+      @client = TwitterClient.new(&block)
+    end
+
+    def rejects=(*args)
+      @rejects = args.flatten
+    end
+
+    def include_hashtags=(boolean)
+      not_a_boolean_error(boolean)
+      @include_hashtags = boolean
+    end
+
+    def include_uris=(boolean)
+      not_a_boolean_error(boolean)
+      @include_uris = boolean
+    end
+    alias include_urls= include_uris=
+
+    def include_mentions=(boolean)
+      not_a_boolean_error(boolean)
+      @include_mentions = boolean
+    end
+
+    def up_to(&time_block)
+      @up_to_block = time_block
+    end
+
+    def up_to_time
+      up_to_block.call.to_time
+    end
+
+    private
+
+    # private method
+    def set_defaults
+      ivars = %i[include_uris include_hashtags include_mentions range client up_to_block rejects]
+      ivars.each { |ivar| instance_variable_set("@#{ivar}", DEFAULT_OPTIONS[ivar]) }
+    end
+
+    # private method
+    def a_boolean?(other)
+      [true, false].include?(other)
+    end
+
+    # private method
+    def not_a_boolean_error(boolean)
+      raise ArgumentError, 'argument must be a booolean value' unless a_boolean?(boolean)
+    end
+  end
+end

data/lib/twords/instance_methods.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'uri'
+
+require 'twords/config_accessible'
+require 'twords/word_matcher'
+
+# Instance methods
+class Twords
+  include ConfigAccessible
+
+  attr_reader :screen_names, :words
+
+  def initialize(*screen_names)
+    @screen_names = screen_names.flatten
+    @words        = {}
+  end
+
+  def audited?
+    @audited
+  end
+
+  def audit
+    count_words unless audited?
+    @audited = true
+  end
+
+  def audit!
+    instance_variables.reject { |ivar| %i[@screen_names @words].include?(ivar) }.each do |ivar|
+      instance_variable_set(ivar, nil)
+    end
+
+    audit
+  end
+
+  def sort_words
+    @_sort_words ||= words.sort { |a, b| b.last <=> a.last }
+  end
+  alias words_forward sort_words
+
+  def tweets
+    @_tweets ||= client.filter_tweets(screen_names)
+  end
+
+  def sort_tweets
+    tweets.sort { |a, b| b.created_at <=> a.created_at }
+  end
+
+  def sort_tweets!
+    tweets.sort! { |a, b| b.created_at <=> a.created_at }
+  end
+
+  def tweets_count
+    @_tweets_count ||= tweets.count
+  end
+
+  def total_word_count
+    @_total_word_count ||= words.values.reduce(:+)
+  end
+
+  def percentages
+    @_percentages ||= words.each_with_object({}) do |word_count, hash|
+      hash[word_count.first] = percentage(word_count.last)
+    end
+  end
+
+  def sort_percentages
+    @_sort_percentages ||= percentages.sort { |a, b| b.last <=> a.last }
+  end
+
+  def to_csv
+    CSV.generate do |csv|
+      csv << %w[word count]
+      sort_words.each do |word_count|
+        csv << word_count
+      end
+    end
+  end
+
+  def write_to_csv(opts = {})
+    filename = opts.fetch(:filename) { 'twords_report.csv' }
+    write_file(filename, :to_csv, opts)
+  end
+
+  def to_json
+    sort_words.to_h.to_json
+  end
+
+  def write_to_json(opts = {})
+    filename = opts.fetch(:filename) { 'twords_report.json' }
+    write_file(filename, :to_json, opts)
+  end
+
+  private
+
+  # private method
+  def client
+    config.client
+  end
+
+  # private method
+  def count_words
+    words.clear
+    tweets.each do |tweet|
+      words_array(tweet).each do |word|
+        next if WordMatcher.should_be_skipped?(word)
+        words.key?(word) ? words[word] += 1 : words[word] = 1
+      end
+    end
+  end
+
+  # private method
+  def words_array(tweet)
+    tweet.attrs[:full_text].downcase.split(' ')
+  end
+
+  # private method
+  def percentage(count)
+    (count / total_word_count.to_f * 100)
+  end
+
+  # private method
+  def write_file(filename, method, opts = {})
+    File.open(filename, 'w', opts) { |file| file.write send(method) }
+  end
+end

data/lib/twords/twitter_client.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal
+
+require 'twords/config_accessible'
+
+class Twords
+  # Twitter REST API client
+  class TwitterClient
+    include ConfigAccessible
+
+    attr_reader :client
+
+    def initialize(&block)
+      @client = Twitter::REST::Client.new(&block)
+    end
+
+    def filter_tweets(screen_names)
+      full_timeline(screen_names).each_with_object([]) do |tweet, memo|
+        next if tweet.created_at > up_to_time
+        memo << tweet if age_of_tweet_in_days(tweet) <= range
+      end
+    end
+
+    private
+
+    # private method
+    def full_timeline(screen_names)
+      screen_names.map { |screen_name| fetch_user_timeline(screen_name) }.flatten.uniq
+    end
+
+    # private method
+    def fetch_user_timeline(screen_name)
+      return [] if screen_name.to_s.empty?
+      user_timeline = client.user_timeline(screen_name, tweet_mode: 'extended', count: 200)
+      return user_timeline if user_timeline.empty?
+      user_timeline = fetch_older_tweets(user_timeline, screen_name)
+      puts "Fetched #{screen_name}'s timeline"
+      user_timeline
+    rescue Twitter::Error::TooManyRequests
+      puts 'Rate limit exceeded, waiting 5 minutes' && sleep(300)
+      fetch_user_timeline(screen_name)
+    end
+
+    # private method
+    def age_of_tweet_in_days(tweet)
+      (up_to_time - tweet.created_at) / 86_400
+    end
+
+    # private method
+    def up_to_time
+      config.up_to_time
+    end
+
+    # private method
+    def range
+      config.range
+    end
+
+    # private method
+    def fetch_older_tweets(user_timeline, screen_name)
+      return user_timeline if age_of_tweet_in_days(user_timeline.last) > range
+      first_count = user_timeline.count
+      user_timeline += client.user_timeline(
+        screen_name,
+        tweet_mode: 'extended',
+        max_id: user_timeline.last.id - 1,
+        count: 200
+      )
+      return user_timeline if user_timeline.count == first_count
+      fetch_older_tweets(user_timeline, screen_name)
+    end
+  end
+end

data/lib/twords/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Twords
-  VERSION = '0.1.8'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/lib/twords/word_matcher.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'twords/config_accessible'
+
+class Twords
+  # Checks if words should be counted or not
+  class WordMatcher
+    class << self
+      include ConfigAccessible
+
+      def should_be_skipped?(word)
+        reject?(word) || hashtag?(word) || uri?(word) || mention?(word)
+      end
+
+      def reject?(word)
+        config.rejects.include?(word)
+      end
+
+      def hashtag?(word)
+        return if config.include_hashtags
+        !(word =~ /#(\w+)/).nil?
+      end
+
+      def uri?(word)
+        return if config.include_uris
+        !(word =~ URI.regexp).nil?
+      end
+
+      def mention?(word)
+        return if config.include_mentions
+        !(word =~ /@(\w+)/).nil?
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twords
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - M. Simon Borg
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-22 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
@@ -62,7 +62,12 @@ files:
 - LICENSE.txt
 - README.md
 - lib/twords.rb
+- lib/twords/config_accessible.rb
+- lib/twords/configuration.rb
+- lib/twords/instance_methods.rb
+- lib/twords/twitter_client.rb
 - lib/twords/version.rb
+- lib/twords/word_matcher.rb
 - twords.gemspec
 homepage: https://github.com/msimonborg/twords
 licenses: