socrates_scraper 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YmFmZDIxNDVmOGFhYTI5ODY1NDQ5YmRmMWMwNDczZjNjNzU0NTkzYQ==
+  data.tar.gz: !binary |-
+    NmFmYTRhYWZmZjI4YmNlZmQyNGE2NDIyOTVkZTE4ZjFlZjE4ZWUwYw==
+SHA512:
+  metadata.gz: !binary |-
+    NjdjMTE0ODVkZjQ0ZDQ2NWJhNzY2Yzk1ZTFkNzYzMGMxY2RhZDMyNGUxYjk2
+    ZDlkYTI4N2JmZWUyMjA0YjBmMTM4ZDIxZjFiMTQ0YmYzMjYxY2UzODBhZWNl
+    ZDczMGMxNGMyNGE4ZjBjYjQxMDg0Y2U4ZjZmNjE1NmIwOWUyYjQ=
+  data.tar.gz: !binary |-
+    YjliMDU0YTY1NmUwZjQ2Y2RlNjQxNTdlMTNhNjdiNDExZTUxYzFhZTJhOWM5
+    ZjdjM2MzODFjOTEwMWM0Y2ZlNzIyOGIzYTk5MDk3YzBmZTBiOWMzYTEzMDRh
+    M2RmNTdlNjY5YmNhMGJhOTQwOTlkMTBiMjM4ZjY0M2Y5Mjc5ZjY=

data/.gitignore

@@ -0,0 +1,2 @@
+*.html
+*.css

data/gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in api_roulette.gemspec
+gemspec

data/lib/socrates_scraper.rb

@@ -0,0 +1,2 @@
+require_relative 'socrates_scraper/scraper'
+require_relative 'socrates_scraper/student'

data/lib/socrates_scraper/scraper.rb

@@ -0,0 +1,88 @@
+require 'mechanize'
+
+class Scraper
+	URL = "https://socrates.devbootcamp.com/login"
+
+	def initialize(email, password)
+		@email = email
+		@password = password
+
+		@agent = Mechanize.new
+	end
+
+	def get_user_profiles
+		cohort_page = get_cohort_page
+		user_links = get_user_links(cohort_page)
+
+		raise "Invalid username or password" if user_links.empty?
+
+		puts "Getting user profiles..."
+		create_user_profiles(user_links)
+	end
+
+	private
+
+	def get_cohort_page
+		@agent.get(URL)
+		login
+		@agent.get("https://socrates.devbootcamp.com/cohorts/78")
+	end
+
+	def login
+		form = @agent.page.forms.first
+		form.email = @email
+		form.password = @password
+		form.submit
+	end
+
+	def get_user_links(page)
+		users = page.links_with(href: /users/)
+
+		# remove top link 'My Profile'
+		users.shift
+
+		# deduplicate links (photo link + text link)
+		users.uniq { |link| link.href }
+	end
+
+	def create_user_profiles(user_links)
+		user_links.map do |link|
+			# reset to cohort page after every user
+			@agent.transact do
+				user_html = extract_profile_html(link)
+				convert_html_to_person_hash(user_html)
+			end
+		end
+	end
+
+	def extract_profile_html(link)
+		page = @agent.click(link)
+
+		# get the socrates id and convert into an
+		# easily identifiable html element
+		socrates_id = link.href[/.*\/(\d*)/, 1]
+		socrates_tag = "<p class='soc_id'>#{socrates_id}</p>"
+
+		Nokogiri::HTML(page.search('div.profile').to_s << socrates_tag)
+	end
+
+	def convert_html_to_person_hash(html)
+		person = {}
+		person[:name] = get_name(html)
+		person[:image] = get_image(html)
+		person[:facebook_id] = get_facebook_id(html)
+		person
+	end
+
+	def get_name(html)
+		html.css('h1 > text()').first.text.lstrip.rstrip
+	end
+
+	def get_image(html)
+		html.css('.user > img').first['src']
+	end
+
+	def get_facebook_id(html)
+		html.css('dd')[4].text[/.*\/(.*)/, 1]
+	end
+end

data/lib/socrates_scraper/student.rb

@@ -0,0 +1,9 @@
+class Student
+	attr_reader :name, :facebook_id, :image
+
+	def initialize(args)
+    @name = args.fetch(:name)
+    @facebook_id = args.fetch(:facebook_id)
+    @image = args.fetch(:image)
+	end
+end

data/socrates_scraper.gemspec

@@ -0,0 +1,25 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name        = 'socrates_scraper'
+  spec.version     = '0.0.1'
+  spec.date        = '2014-08-22'
+  spec.summary     = "Get user info from a Socrates cohort"
+  spec.description = "Scrapes user info from a cohort in Socrates"
+  spec.authors     = ["James Robinson"]
+  spec.email       = 'james.michael.robinson@gmail.com'
+  spec.files       = ["lib/facebook_word_counter.rb"]
+  spec.homepage    = 'http://rubygems.org/gems/facebook_word_counter'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "nokogiri", '~> 1.6'
+  spec.add_runtime_dependency 'mechanize', '~> 2.7.3'
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: socrates_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- James Robinson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.3
+description: Scrapes user info from a cohort in Socrates
+email: james.michael.robinson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- gemfile
+- lib/socrates_scraper.rb
+- lib/socrates_scraper/scraper.rb
+- lib/socrates_scraper/student.rb
+- socrates_scraper.gemspec
+homepage: http://rubygems.org/gems/facebook_word_counter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.5
+signing_key: 
+specification_version: 4
+summary: Get user info from a Socrates cohort
+test_files: []