genderstat 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9d948276334bd55efb359df9eb7b3fb86dbfc3c1
+  data.tar.gz: a462a1c66d02849eca80ade99bfc5eeec7fa7d6b
+SHA512:
+  metadata.gz: 123eca8ae0ded0774bcdfbbd4fb1754f6db043449e910ffebda9f1bcd835f3cad5d9df97ebfb2a93b28b0cde98298a8a833f675b260530f24ba4401f59073ec1
+  data.tar.gz: 90d1988ad9bbb61896d08bf9fe31079e16064dfe5bdcad29c1f656904056dfc6abfde7715e81b01349892b2c435e185e30f5faff83e86f72b24e5f6fa67d16ee

data/.gitignore

@@ -0,0 +1,3 @@
+*.txt
+Gemfile.lock
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'open_uri_redirections'

data/README.md

@@ -0,0 +1,133 @@
+genderstat
+===
+genderstat calculates the amount (in terms of percentages and relative ratios) of gendered language in a file or on a site.
+
+Specifically, it counts the number of gendered:
+- subject pronouns (she/he)
+- object pronouns (her/him)
+- possessives (hers/his)
+- generic nouns (woman/man, girl/boy, womankind/mankind, etc)
+
+It does so both in absolute numbers, and also calculates the percent.
+
+Additionally, it offers the ratios of between each too (e.g. 3.2 times as many
+masculine words as feminine words).
+
+Word Lists
+---
+genderstat comes with three wordlists:
+- ```masculine_words.yaml```
+- ```feminine_words.yaml```
+- ```neutral_words.yaml```
+
+They're completely editable, and genderstat will calculate for any files in the
+directory names ending with ```_words.yaml```
+
+
+Installation
+-------
+
+`gem install genderstat`
+
+Usage
+---
+
+`genderstat [FILE]`
+
+or
+
+`genderstat [URL]`
+
+
+### File example ###
+
+The `ralph-waldo-emerson.txt` file is a collection of his essays and `kate-chopin.txt` is a collection of her stories (including The Awakening).
+```
+    % ruby -Ilib bin/genderstat ralph-waldo-emerson.txt
+    total words: 77181
+    feminine words: 85
+    masculine words: 1942
+    neutral words: 1714
+
+    feminine words: 0.11%
+    masculine words: 2.52%
+    neutral words: 2.22%
+
+    The ratio of feminine to masculine words is 0.04
+    The ratio of feminine to neutral words is 0.05
+    The ratio of masculine to feminine words is 22.85
+    The ratio of masculine to neutral words is 1.13
+    The ratio of neutral to feminine words is 20.16
+    The ratio of neutral to masculine words is 0.88
+
+
+    % ruby -Ilib bin/genderstat kate-chopin.txt
+    total words: 67023
+    feminine words: 3147
+    masculine words: 1724
+    neutral words: 1095
+
+    feminine words: 4.7%
+    masculine words: 2.57%
+    neutral words: 1.63%
+
+    The ratio of feminine to masculine words is 1.83
+    The ratio of feminine to neutral words is 2.87
+    The ratio of masculine to feminine words is 0.55
+    The ratio of masculine to neutral words is 1.57
+    The ratio of neutral to feminine words is 0.35
+    The ratio of neutral to masculine words is 0.64
+```
+
+### URL example ###
+```
+    % ruby -Ilib bin/genderstat feministing.com
+    total words: 4908
+    feminine words: 20
+    masculine words: 3
+    neutral words: 17
+
+    feminine words: 0.41%
+    masculine words: 0.06%
+    neutral words: 0.35%
+
+    The ratio of feminine to masculine words is 6.67
+    The ratio of feminine to neutral words is 1.18
+    The ratio of masculine to feminine words is 0.15
+    The ratio of masculine to neutral words is 0.18
+    The ratio of neutral to feminine words is 0.85
+    The ratio of neutral to masculine words is 5.67
+
+    %ruby -Ilib bin/genderstat stallman.org
+    total words: 5673
+    feminine words: 1
+    masculine words: 16
+    neutral words: 55
+
+    feminine words: 0.02%
+    masculine words: 0.28%
+    neutral words: 0.97%
+
+    The ratio of feminine to masculine words is 0.06
+    The ratio of feminine to neutral words is 0.02
+    The ratio of masculine to feminine words is 16.0
+    The ratio of masculine to neutral words is 0.29
+    The ratio of neutral to feminine words is 55.0
+    The ratio of neutral to masculine words is 3.44
+```
+todo
+---
+- Add tests
+- Add support for more stats (like statistical significance?)
+- Add support for reading from stdin
+- Handle scenario of https->redirect (rather than weird nil error)
+
+Dependencies
+------------
+- Ruby 2.0+
+- ```open_uri_redirections``` gem
+
+You can get the gem by running ```bundle install```.
+
+It's not strictly necessary, it's just allows for HTTP->HTTPS redirections.
+

data/bin/genderstat

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby -I ../lib
+
+require 'genderstat'
+
+if ARGV.length != 1
+  abort("Usage: genderstat [FILE] or genderstat [URL]")
+end
+
+genderstat = Genderstat.new(ARGV[0])
+genderstat.calculate
+genderstat.print_all_results

data/feminine_words.yaml

@@ -0,0 +1,11 @@
+- she
+- her
+- hers
+- she's
+- herself
+- woman
+- lady
+- girl
+- womankind
+- women
+- ladies

data/genderstat.gemspec

@@ -0,0 +1,17 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "genderstat/version"
+
+Gem::Specification.new do |s|
+  s.name                  = 'genderstat'
+  s.version               = Genderstat::VERSION
+  s.date                  = '2014-08-23'
+  s.summary               = "A gendered language frequency calculator"
+  s.description           = "Calculate the relative frequencies of gendered language in a file or on the web"
+  s.authors               = ["Sean Collins"]
+  s.email                 = 'sean@cllns.com'
+  s.files                 = `git ls-files`.split($/)
+  s.executables           = 'genderstat'
+  s.required_ruby_version = '>= 2.0'
+  s.homepage              = 'http://github.org/cllns/genderstat'
+  s.license               = 'MIT'
+end

data/lib/genderstat.rb

@@ -0,0 +1,48 @@
+#! /usr/bin/ruby
+# A program to determine how "masculine" a piece of writing is.
+# Sean Collins 11/11/12
+require 'yaml'
+require 'genderstat/text_reader'
+require 'genderstat/word_counters'
+
+class Genderstat
+  DECIMAL_DIGITS_OF_PRECISION = 2
+
+  def initialize arg
+    @word_counters = WordCounters.new
+    @text_reader = TextReader.new arg
+  end
+
+  def calculate
+    @all_words = @text_reader.read
+    @all_words.each { |word| @word_counters.check(word) }
+  end
+
+  def print_all_results
+    puts "total words: #{@all_words.count.to_s}"
+    print_totals
+    print_percentages
+    print_ratios
+  end
+
+  def print_totals
+    @word_counters.get_totals.each do |name, count|
+      puts "#{name} words: #{count}"
+    end
+    puts
+  end
+
+  def print_percentages
+    @word_counters.get_percentages(@all_words.count).each do |name, percentage|
+      puts "#{name} words: #{percentage}%"
+    end
+    puts
+  end
+
+  def print_ratios
+    @word_counters.get_ratios.each do |name, ratio|
+      puts "The ratio of #{name.gsub("_", " ")} words is #{ratio}"
+    end
+    puts
+  end
+end

data/lib/genderstat/file_reader.rb

@@ -0,0 +1,9 @@
+class FileReader
+  def initialize file_name
+    @file_name = file_name
+  end
+
+  def read
+    IO.read @file_name
+  end
+end

data/lib/genderstat/text_reader.rb

@@ -0,0 +1,16 @@
+require 'genderstat/file_reader'
+require 'genderstat/web_reader'
+
+class TextReader
+  def initialize file_locator
+    if File.exist? file_locator
+      @reader = FileReader.new file_locator
+    else
+      @reader = WebReader.new file_locator
+    end
+  end
+
+  def read
+    @reader.read.downcase.split
+  end
+end

data/lib/genderstat/version.rb

@@ -0,0 +1,3 @@
+module Genderstat
+  VERSION = '0.0.2'
+end

data/lib/genderstat/web_reader.rb

@@ -0,0 +1,35 @@
+require 'open-uri'
+require 'open_uri_redirections'
+require 'socket'
+
+class WebReader
+  def initialize url
+    @url = clean_up_url url
+  end
+
+  def read
+    begin
+      open(@url, :allow_redirections => :safe).read
+    rescue Exception => ex
+      handle_web_exceptions ex
+    end
+  end
+
+  private
+  def handle_web_exceptions ex
+    if ex == OpenURI::HTTPError || ex == SocketError
+      abort "Could not open: #{url}"
+    else
+      abort ex.to_s
+    end
+  end
+
+  # if the URL is doesn't match the URI regex, then prepend it with http://
+  def clean_up_url url
+    if (url =~ URI.regexp).nil?
+      "http://#{url}"
+    else
+      url
+    end
+  end
+end

data/lib/genderstat/word_counter.rb

@@ -0,0 +1,15 @@
+require 'set'
+
+class WordCounter
+  attr_reader :count, :name
+
+  def initialize filename
+    @words = Set.new YAML.load_file(filename)
+    @name = filename.split('_').first.split('/').last
+    @count = 0
+  end
+
+  def is_in_here? word_in_question
+    @count += 1 if @words.include? word_in_question
+  end
+end

data/lib/genderstat/word_counters.rb

@@ -0,0 +1,53 @@
+require 'genderstat/word_counter'
+
+class WordCounters
+  def initialize
+    @word_counters = []
+    # The word lists are two directories higher than this file, so that's
+    # how we have to reference their locations
+    this_file_path = File.dirname(__FILE__)
+    word_list_relative_location = "../../*_words.yaml"
+    word_list_location = File.join(this_file_path, word_list_relative_location)
+
+    Dir.glob(word_list_location).each do |filename|
+      @word_counters << WordCounter.new(filename)
+    end
+  end
+
+
+  def check word
+    @word_counters.each { |wc| wc.is_in_here? word }
+  end
+
+  def get_totals
+    @word_counters.each_with_object({}) do |wc, hash|
+      hash[wc.name] = wc.count
+    end
+  end
+
+  def get_percentages total_word_count
+    @word_counters.each_with_object({}) do |wc, hash|
+      hash[wc.name] = round( 100 * (wc.count.to_f / total_word_count))
+    end
+  end
+
+  # We get the ratios of counts among all word_lists
+  # This returns a hash with keys that are named "name_to_other_name"
+  # This loop does twice as much work as it needs to, since the ratios
+  #  are reciprocals of one another, but oh well! It's only division.
+  def get_ratios
+    @word_counters.each_with_object({}) do |wc, hash|
+      # We skip over the ratio of name_to_name, since it'll always be one
+      @other_word_counters = @word_counters - [wc]
+      @other_word_counters.each do |other_wc|
+        ratio = round(( wc.count.to_f / other_wc.count.to_f) )
+        hash["#{wc.name}_to_#{other_wc.name}"] = ratio
+      end
+    end
+  end
+
+  def round float
+    float.round(Genderstat::DECIMAL_DIGITS_OF_PRECISION)
+  end
+
+end

data/masculine_words.yaml

@@ -0,0 +1,11 @@
+- he
+- him
+- his
+- he's
+- himself
+- man
+- dude
+- boy
+- mankind
+- men
+- dudes

data/neutral_words.yaml

@@ -0,0 +1,9 @@
+- it
+- it's
+- its
+- they
+- their
+- someone
+- anyone
+- somebody
+- someone

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: genderstat
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Sean Collins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-23 00:00:00.000000000 Z
+dependencies: []
+description: Calculate the relative frequencies of gendered language in a file or
+  on the web
+email: sean@cllns.com
+executables:
+- genderstat
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- bin/genderstat
+- feminine_words.yaml
+- genderstat.gemspec
+- lib/genderstat.rb
+- lib/genderstat/file_reader.rb
+- lib/genderstat/text_reader.rb
+- lib/genderstat/version.rb
+- lib/genderstat/web_reader.rb
+- lib/genderstat/word_counter.rb
+- lib/genderstat/word_counters.rb
+- license.txt
+- masculine_words.yaml
+- neutral_words.yaml
+homepage: http://github.org/cllns/genderstat
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: A gendered language frequency calculator
+test_files: []