facebook_word_counter 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MWE4YmU0ZmVmZWIzOTJjMmJiYTAxMWEzOWZjYjk5YzdkZmE3ZjQ0OQ==
+  data.tar.gz: !binary |-
+    N2NiNGRlZjBhODIyMGQ1OTc3NmQ5NmRmYmI4MzZkMzg2NTMwNjk3Yg==
+SHA512:
+  metadata.gz: !binary |-
+    Nzk3MGNhYzEwNjUyMjIyZmEwMjY2N2UyOTBjZjNhNTRlYTRmMzE1YjJiZTYw
+    MmU3MDkxOGZmOTdhYjRiNWUzMTA0YTk3NWY3YzQxZjc4ODQwN2I1YzNkZjQx
+    OTAxN2NjN2NmYTY2OGY5MTNhMzk5NWVhMDE2MmJjZGYxZTYxZGQ=
+  data.tar.gz: !binary |-
+    ZDJjYTNkYzUzYzQwM2IwMjYyN2FkNTRjZGRkYWY1OGY3NDdlYjhhMGQwNTVl
+    YWMyNGMwMGI5YmRiZDNhNzg4MTY1NjZiNGQ3ZDllZTJlZDE0M2JjMTY4NTM5
+    M2UwM2U1YTU3NTRmMDE2ODZkMjdkZDc4NjIxMGNhYmVhZDAyMjM=

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    facebook_word_counter (0.0.1)
+      mechanize (~> 2.7.3)
+      nokogiri (~> 1.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    domain_name (0.5.20)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (2.3)
+    mini_portile (0.6.0)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    ntlm-http (0.1.1)
+    rake (10.3.2)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  facebook_word_counter!
+  rake

data/facebook_word_counter.gemspec

@@ -0,0 +1,25 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name        = 'facebook_word_counter'
+  spec.version     = '0.0.1'
+  spec.date        = '2014-08-21'
+  spec.summary     = "Get word counts from Facebook pages"
+  spec.description = "Given a list of facebook ids, scrapes the front page of posts and returns a hash of word counts"
+  spec.authors     = ["James Robinson"]
+  spec.email       = 'james.michael.robinson@gmail.com'
+  spec.files       = ["lib/facebook_word_counter.rb"]
+  spec.homepage    = 'http://rubygems.org/gems/facebook_word_counter'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "nokogiri", '~> 1.6'
+  spec.add_runtime_dependency 'mechanize', '~> 2.7.3'
+end

data/gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in api_roulette.gemspec
+gemspec

data/lib/facebook_word_counter/facebook_word_counter.rb

@@ -0,0 +1,79 @@
+require 'nokogiri'
+require 'mechanize'
+
+class NotLoggedInError < StandardError; end
+
+class FacebookWordCounter
+	FACEBOOK_URL_PREFIX = "https://www.facebook.com/"
+	FACEBOOK_URL_SUFFIX = "?_fb_noscript=1"
+
+	def self.setup_browser_agent(username, password)
+		@agent = Mechanize.new
+		@agent.get(FACEBOOK_URL_PREFIX)
+		login(username, password)
+	end
+
+	def self.get_word_counts(*facebook_pages)
+		raise NotLoggedInError unless logged_in?
+
+		facebook_pages.map do |url|
+			no_js_url = "#{FACEBOOK_URL_PREFIX}#{url}#{FACEBOOK_URL_SUFFIX}"
+			@agent.get(no_js_url)
+			{
+				name: get_facebook_name,
+				image: get_facebook_photo_url,
+				# facebook_url: "#{FACEBOOK_URL_PREFIX}#{url}",
+				words: get_word_counts_on_page
+			}
+		end
+	end
+
+	private
+
+	def self.login(username, password)
+		form = @agent.page.forms.first
+		form.email = username
+		form.pass = password
+		form.submit
+	end
+
+	def self.logged_in?
+		!(@agent.page.uri.to_s =~ /login_attempt/)
+	end
+
+	def self.get_facebook_name
+		@agent.page.title
+	end
+
+	def self.get_word_counts_on_page
+		all_words = @agent.page.search(".userContent").map { |post| clean_text(post.text) }.join(' ')
+		WordCounter.count_words(all_words)
+	end
+
+	def self.get_facebook_photo_url
+		@agent.page.search("img.profilePic").first['src']
+	end
+
+	def self.clean_text(text)
+		text = remove_urls(text.downcase)
+		text = remove_punctuation(text)
+		text = remove_filler_words(text)
+		text.strip
+	end
+
+	def self.remove_urls(text)
+		text.gsub(/https?:\/\/[\S]+/, '')
+	end
+
+	def self.remove_punctuation(text)
+		text.gsub(/[^a-zA-Z\s]/, '')
+	end
+
+	FILLER_WORDS = %w{ and the of as if is to in my a be will i ive are not my are not can out go am im for }
+	def self.remove_filler_words(text)
+		FILLER_WORDS.each do |word|
+			text.gsub!(/ #{word} / , ' ')
+		end
+		text
+	end
+end

data/lib/facebook_word_counter/word_counter.rb

@@ -0,0 +1,8 @@
+class WordCounter
+	def self.count_words(words)
+		words.split(' ').reduce(Hash.new(0)) do |word_counts, word|
+			word_counts[word] += 1
+			word_counts
+		end
+	end
+end

data/lib/facebook_word_counter.rb

@@ -0,0 +1,2 @@
+require_relative './facebook_word_counter/facebook_word_counter'
+require_relative './facebook_word_counter/word_counter'

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: facebook_word_counter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- James Robinson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.3
+description: Given a list of facebook ids, scrapes the front page of posts and returns
+  a hash of word counts
+email: james.michael.robinson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile.lock
+- README.md
+- facebook_word_counter.gemspec
+- gemfile
+- lib/facebook_word_counter.rb
+- lib/facebook_word_counter/facebook_word_counter.rb
+- lib/facebook_word_counter/word_counter.rb
+homepage: http://rubygems.org/gems/facebook_word_counter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.5
+signing_key: 
+specification_version: 4
+summary: Get word counts from Facebook pages
+test_files: []