beauvoir 0.0.1a → 0.0.2b

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c046bac3fbc6a13b9f585aecaa4c03fa257e66f
-  data.tar.gz: a47a01be8e526d0deefab9c1bf7d89c8ac99f5d4
+  metadata.gz: d08d58653a1e5586213426ba0026787bb0c9d5c6
+  data.tar.gz: 65f2d360cec83211db9809f6025195f243737720
 SHA512:
-  metadata.gz: c61d84c385ca7c556d052c6be4788c55e39a852a9ec324d5e00c7becb1c4e66ae8ba71d1f6b5043a799d6b6061a3500a4fa6222811916cb5fd28546592b0045c
-  data.tar.gz: 57da2d4ada7c580d07715713d7296bbb7c68ea049affe8339a7c535c852264040a81f4229b870e12271d5cc39103a0bde15f37ee4927a848fc28d8aa475764b1
+  metadata.gz: 495d9a48b07c37cb9110f4787cc75b8c4b9bc79089c34e493d541f09514223708ffabb995d199e9fd786b7a03307f0f3e73616e5a0e691609cab1c42563c8b7e
+  data.tar.gz: 189b378c9f7eacee1acaf206a60d975fd9f1c4108eb2b8cbad53a4535c7154f74a6398fea2ae2f66a0f2f1a90466170b6bd147d3e7e654d220fc10208b24be7b

data/README.md

@@ -5,6 +5,10 @@ Beauvoir is a gem for guessing a person's gender by their first name. Caveats ap
 
 Beauvoir uses more precise data sourced from [Open Gender Tracker](http://opengendertracking.org/)'s [Global Name Data](https://github.com/OpenGenderTracking/globalnamedata). Beauvoir lets you set avg and lower bounds and choose countries from which to draw data (so far US, UK only, more to come soon).
 
+Caution
+-------
+This is pre-alpha software. The API will change, I guarantee it.
+
 Caveats
 -------
 
@@ -56,6 +60,14 @@ Methodology
 -----------
 to come soon. email me if you need it asap.
 
+A minimum estimated value: a best guess of the ratio of genders of people with a given name.
+A minimum lower confidence bound: only 2.5 times out of a hundred (by default) with the _actual_ proportion of genders of people with this name fall below this bound. (It will be above outside the confidence interval 5% of the time, half of which is above and half of which is below. Ninety-five percent of the time, it will fall within the confidence interval. )
+Set a level of statistical significance -- by default 0.95. The lower this level, the more likely the "true" parameter will fall outside the interval.
+
 Name
 ----
-Beauvoir is named after [Simone de Beauvoir](http://en.wikipedia.org/wiki/Simone_de_Beauvoir), a feminist author best known for _The Second Sex), which Wikipeida describes as "a detailed analysis of women's oppression and a foundational tract of contemporary feminism." Hopefully Beauvoir (the Gem) will have some effect by assisting computational efforts to shed light on gender bias and discrimination.
+Beauvoir is named after [Simone de Beauvoir](http://en.wikipedia.org/wiki/Simone_de_Beauvoir), a feminist author best known for _The Second Sex), which Wikipedia describes as "a detailed analysis of women's oppression and a foundational tract of contemporary feminism." Hopefully Beauvoir (the Gem) will have some effect by assisting computational efforts to shed light on gender bias and discrimination.
+
+TODO
+----
+- Test A.C. Confidence Interval, Expected Value results against the ones in the source data. (The R is abstruse like a goose, so I'm not sure it's implemented right.)

data/lib/beauvoir.rb

@@ -1,25 +1,24 @@
 require 'csv'
 require 'set'
-require './lib/name'
+require_relative './name'
 
 class Beauvoir
-  DEFAULT_AVERAGE_CONFIDENCE_THRESHOLD = 0.99
-  DEFAULT_LOWER_CONFIDENCE_THRESHOLD = 0.5
+  DEFAULT_PROPORTION_THRESHOLD = 0.99
+  DEFAULT_LOWER_CONFIDENCE_BOUND = 0.75
+
 
   # these aren't writable because once a Beauvoir is initialized, since their
   # value is baked into Beauvoir's internal judgments of gender.
-  attr_reader :threshold, :lower_threshold, :names_by_names, :names_gendersh
+  attr_reader :threshold, :lower_confidence_bound, :names_by_names, :names_genders
 
   def initialize(options={})
-
-
     countries = Set.new([:us, :uk])
 
-    @threshold = options[:threshold] || DEFAULT_AVERAGE_CONFIDENCE_THRESHOLD
+    @threshold = options[:threshold] || DEFAULT_PROPORTION_THRESHOLD
 
     # TODO: what should this be in the default case? (0, i.e. ignore the lower bound?, some sensical value to
     # exclude a naive user from getting back nonsense? the bare minimum value for a loose significance level?)
-    @lower_threshold = options[:lower_threshold] || DEFAULT_LOWER_CONFIDENCE_THRESHOLD
+    @lower_confidence_bound = options[:lower_confidence_bound] || DEFAULT_LOWER_CONFIDENCE_BOUND
 
     @names_by_names = {}
     # @country_totals = {}
@@ -37,11 +36,13 @@ class Beauvoir
     # to avoid ~10sec delay when loading into memory
     #(e.g. seeking around the file?)
     countries.each do |country|
-      CSV.open("lib/data/#{country}processed.csv", :headers => true).each do |row|
-        name = @names_by_names.fetch(row["Name"], Name.new(row["Name"]))
+
+      CSV.open(File.join(File.dirname(File.expand_path(__FILE__)), "data/#{country}processed.csv"), :headers => true).each do |row|
+        name_str = Beauvoir.normalize(row["Name"])
+        name = @names_by_names.fetch(name_str, Name.new(name_str))
         name.male_count += row["count.male"].to_i
         name.female_count += row["count.female"].to_i
-        @names_by_names[row["Name"]] = name
+        @names_by_names[name_str] = name
       end
     end
 
@@ -52,20 +53,25 @@ class Beauvoir
                                     :unknown
                                   end
     end
-    raise NoJohnError unless @names_by_names.include?("John")
     self
   end
 
   def sufficiently_confident(name)
     (name.male_proportion > @threshold || name.female_proportion > @threshold) &&
-      name.lower > @lower_threshold
+      name.lower > @lower_confidence_bound
   end
 
   #
-  # Transform any name-like string into an unpadded, initial-cased first name
+  # Transform any name-like string into an unpadded, initial-cased first name.
+  # Should be a surjection, mapping many possible inputs (e.g. "Jeremy", "Jeremy.", "JEREMY", "Jeremy B. Merrill")
+  # onto one single name.
+  # This is used for two things:
+  # 1. Accepting differently-formatted/tokenized names from the user.
+  # 2. Dealing with differently-formatted names from the source agencies (e.g. "Mckinley" v. "McKinley", "Obrien", vs. "O'brien")
   #
-  def normalize(name)
-    name.gsub!(/[^A-Za-z \-\']+/, '') #TODO: can this be done more efficiently with String#tr?
+  def self.normalize(name)
+    name.tr!("^A-Za-z' \-", '')
+    # name.gsub!(/[^A-Za-z \-\']+/, '') #this I suspect is done more efficiently with String#tr
     if name.include?(" ")
       name = name[0...name.index(" ")]
     end
@@ -73,18 +79,34 @@ class Beauvoir
   end
 
   def guess(name)
-    @names_genders.fetch(normalize(name), :unknown)
+    @names_genders.fetch(Beauvoir.normalize(name), :unknown)
+  end
+
+  def estimated_male_value(name)
+    if name_obj = @names_by_names[Beauvoir.normalize(name)]
+      name_obj.estimated_male_value
+    else
+      nil
+    end
+  end
+
+  def estimated_female_value(name)
+    if name_obj = @names_by_names[Beauvoir.normalize(name)]
+      name_obj.estimated_female_value
+    else
+      nil
+    end
   end
 
-  def maleness_ratio(name)
-    if name_obj = @names_by_names[normalize(name)]
+  def male_proportion(name)
+    if name_obj = @names_by_names[Beauvoir.normalize(name)]
       name_obj.male_proportion
     else
       nil
     end
   end
-  def femaleness_ratio(name)
-    if name_obj = @names_by_names[normalize(name)]
+  def female_proportion(name)
+    if name_obj = @names_by_names[Beauvoir.normalize(name)]
       name_obj.female_proportion
     else
       nil

data/lib/name.rb

@@ -3,13 +3,20 @@
 class Name
   attr_accessor :male_count, :female_count, :name
 
-  def initialize(name)
+  def initialize(name, options={})
+    # default_options = {
+    #   :significance_level => 0.95,
+    # }
+    @options = options #default_options.merge(options)
+
     @male_count = 0
     @female_count = 0
     @name = name
+    # @significance_level = @options[:significance_level]
   end
 
-  def male?
+  def male? 
+    #pure proportions, so even the slightest greater proportion of one gender will affect this
     @male_count > @female_count
   end
 
@@ -18,7 +25,13 @@ class Name
   end
 
   def gender
-    female? ? :female : male? ? :male : :unknown #oh god this is unreadable
+    if female?
+      :female
+    elsif male?
+      :male
+    else
+      :unknown
+    end
   end
 
   def female_proportion
@@ -35,12 +48,48 @@ class Name
     (@male_count + @female_count).to_f
   end
 
-  #implements Agresti-Coull binomial confidence interval, returns lower bound of higher of male/female
-  # cf. http://codesequoia.wordpress.com/2010/12/06/unit-test-and-statistics/
+
+  # fancy statistics!
+  #------------------
+  # implements Agresti-Coull estimated value and binomial confidence interval
+  # via:
+  # - http://codesequoia.wordpress.com/2010/12/06/unit-test-and-statistics/
+  # - http://stackoverflow.com/questions/3749125/how-should-i-order-these-helpful-scores/3752941#3752941
+  # -
+  #this is apparently related to alpha and related to the level of statistical significance we care about.
+  # 1.96 pertains to a 0.95 significance level.
+  #
+  MAGIC_STATISTICS_NUMBER = 1.96
+
+  def z
+    # TODO: https://github.com/clbustos/statsample/blob/1168d58b14a5095af0a639b4843b31433d40f105/lib/statsample/srs.rb
+    #@significance_level #do stuff with this.
+    MAGIC_STATISTICS_NUMBER
+  end
+
+  def estimated_female_value
+    estimated_value_formula(@female_count)
+  end
+
+  def estimated_male_value
+    estimated_value_formula(@male_count)
+  end
+
+  def estimated_value
+    estimated_value_formula([@male_count, @female_count].max)
+  end
+
+  #  returns lower bound of higher of male/female
   def lower
-    nt = total + 1.96 ** 2
-    pt = ([@male_count, @female_count].max + (1.96 ** 2) / 2) / nt
-    e = 1.96 * Math.sqrt(pt * (1 - pt) / nt)
-    [female_proportion, male_proportion].max - e
+    nt = total + z ** 2
+    interval = z * Math.sqrt(estimated_value * (1 - estimated_value) / nt)
+    [female_proportion, male_proportion].max - interval
+  end
+
+  private
+
+  def estimated_value_formula(observed)
+    nt = total + z ** 2
+    (observed + ((z ** 2) / 2)) / nt
   end
 end

metadata

@@ -1,38 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: beauvoir
 version: !ruby/object:Gem::Version
-  version: 0.0.1a
+  version: 0.0.2b
 platform: ruby
 authors:
 - Jeremy B. Merrill
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-18 00:00:00.000000000 Z
+date: 2013-10-26 00:00:00.000000000 Z
 dependencies: []
-description: |-
-  Guess gender by a first name using more detailed, better
-                      sourced data from Open Gender Tracker's Global Name Data.
-                      Beauvoir lets you set avg and lower bounds and choose
-                      countries from which to draw data.
-
-                      It's important to note that many people identify as neither
-                      a men nor a women. It's important, too, to note that many
-                      people who do identify as male or female have names that
-                      are held by far more people who identify as another gender.
-                      All of these people deserve not to be misgendered in public
-                      (or in private).
-
-                      Nevertheless, automatically classifying people by apparent
-                      gender can be a very useful tool to perform censuses of
-                      communities or publications to detect and quantify
-                      perhaps-invisible bias. VIDA is a pioneer in this field,
-                      but their "Count" is limited by a manual methodology that
-                      depends hundreds of person-hours of labor. There is a place
-                      for more automated counts and Beauvoir can help, but if you
-                      do a count like this, you should be careful in how you word
-                      your findings not to misgender anyone in particular and be
-                      responsive to the possibility of errors.
+description: "Guess gender by a first name using more detailed, better\n                    sourced
+  data from Open Gender Tracker's Global Name Data.<br />\n                    Beauvoir
+  lets you set avg and lower bounds and choose\n                    countries from
+  which to draw data. \n\n\n                    It's important to note that many people
+  identify as neither\n                    a men nor a women. It's important, too,
+  to note that many\n                    people who do identify as male or female
+  have names that<br />\n                    are held by far more people who identify
+  as another gender.\n                    All of these people deserve not to be misgendered
+  in public\n                    (or in private).\n\n                    Nevertheless,
+  automatically classifying people by apparent\n                    gender can be
+  a very useful tool to perform censuses of\n                    communities or publications
+  to detect and quantify\n                    perhaps-invisible bias. VIDA is a pioneer
+  in this field,\n                    but their \"Count\" is limited by a manual methodology
+  that\n                    depends hundreds of person-hours of labor. There is a
+  place\n                    for more automated counts and Beauvoir can help, but
+  if you\n                    do a count like this, you should be careful in how you
+  word\n                    your findings not to misgender anyone in particular and
+  be\n                    responsive to the possibility of errors."
 email: jeremybmerrill@jeremybmerrill.com
 executables: []
 extensions: []
@@ -42,8 +37,8 @@ files:
 - lib/name.rb
 - LICENSE
 - README.md
-- lib/data/ukprocessed.csv
 - lib/data/usprocessed.csv
+- lib/data/ukprocessed.csv
 homepage: http://rubygems.org/gems/beauvoir
 licenses:
 - MIT
@@ -64,9 +59,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Guess a person's gender by their first name
 test_files: []
-has_rdoc: