facebook_profile_scraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6c4e62ff516ff41217241e4633bbc186083ad3f7
+  data.tar.gz: 98b762dcbe415cfe865df7384700b8cbb3dfc48d
+SHA512:
+  metadata.gz: 56458fb203a44aecd9610030e9b5ddb86536b503f33f613f975e672c6162416a11c6e4b6b9a00ca14576fa25109cef900d70fce27684b2d406c9125c06815804
+  data.tar.gz: 8d18c66ad5107f7ec47de168f821571962ee3813789ad7da8f41bc99c31a3cd9ef28c87e511adeed0cb9d1c24116b4d0b717cd73d2170c8af1f93b0399c1a176

data/.gitignore

@@ -0,0 +1,26 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+.byebug_history
+.env
+example.rb

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+ruby '2.3.1'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Maros Hluska
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Facebook Profile Scraper
+
+Scrape your friends' Facebook photos.
+
+## Installation
+
+```
+gem install facebook_profile_scraper
+```
+
+## Usage
+
+Create a `.env` file with your Facebook login information:
+
+```
+FACEBOOK_EMAIL=
+FACEBOOK_PASSWORD=
+```
+
+Use the scraper like this:
+
+```
+require 'facebook_profile_scraper'
+
+scraper = FacebookProfileScraper::Scraper.new
+scraper.scrape('facebook_userid')
+```
+
+The local `tmp` directory will contain album photos.

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+# Handle RSpec not being available on a production environment.
+rescue LoadError
+end
+

data/facebook_profile_scraper.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'facebook_profile_scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'facebook_profile_scraper'
+  spec.version       = FacebookProfileScraper::VERSION
+  spec.authors       = ['Maros Hluska']
+  spec.email         = ['mhluska@gmail.com']
+  spec.summary       = "Scrape a friend's Facebook profile page"
+  spec.description   = "Scrape photos and other data from a friend's Facebook profile page"
+  spec.homepage      = 'http://mhluska.com/'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'selenium-webdriver', '~> 2.53'
+  spec.add_dependency 'dotenv', '~> 2.1'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake', '~> 11.2'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+end

data/lib/facebook_profile_scraper.rb

@@ -0,0 +1,5 @@
+require 'facebook_profile_scraper/version'
+require 'facebook_profile_scraper/scraper'
+
+module FacebookProfileScraper
+end

data/lib/facebook_profile_scraper/scraper.rb

@@ -0,0 +1,141 @@
+require 'capybara'
+require 'capybara/dsl'
+require 'capybara/poltergeist'
+require 'dotenv'
+
+Capybara.configure do |config|
+  config.run_server = false
+  # config.default_driver = :poltergeist
+  config.default_driver = :chrome
+  config.app_host = 'https://www.facebook.com'
+  config.ignore_hidden_elements = false
+end
+
+Capybara.register_driver :chrome do |app|
+  prefs = {
+    download: {
+      prompt_for_download: false,
+      default_directory: "#{Dir.pwd}/tmp/downloads"
+    }
+  }
+  Capybara::Selenium::Driver.new(app, browser: :chrome, prefs: prefs)
+end
+
+# TODO(maros): Figure out how to do this with `poltergeist` instead of
+# `chromedriver`. We use the latter because `poltergeist` can't seem to
+# download images.
+# Capybara.register_driver :poltergeist do |app|
+#   Capybara::Poltergeist::Driver.new(app, phantomjs_logger: '/dev/null', js_errors: false, timeout: 60 * 10)
+# end
+
+Dotenv.load
+
+module FacebookProfileScraper
+  class Scraper
+    include Capybara::DSL
+
+    def scrape(username)
+      login
+      scrape_photos(username)
+    end
+
+    private
+
+    def find_link_elems_with(href)
+      all('a').select { |elem| elem[:href] && elem[:href].include?(href) }
+    end
+
+    def find_links_with(href)
+      find_link_elems_with(href).map { |elem| elem[:href] }.uniq
+    end
+
+    def download_photo
+      tries = 0
+
+      # TODO(maros): Use something like `wait_for` instead.
+      loop do
+        find('.fbPhotoSnowliftDropdownButton').click
+        begin
+          # This will occur if the image can't be downloaded.
+          # TODO(maros): Resort to scraping the smaller res image instead.
+          if tries > 3
+            break
+          end
+
+          elem = find('a[data-action-type="download_photo"]', visible: true)
+        rescue Capybara::ElementNotFound
+          tries += 1
+          sleep 1
+        else
+          elem.click
+          break
+        end
+      end
+    end
+
+    def scrape_album(href)
+      visit(href)
+
+      # HACK(maros): Make the backdrop for Chrome Notifications go away. Find a
+      # pref for `chromedriver` to make this disabled by default.
+      find('._3ixn').click
+
+      photo_links = []
+
+      # Get all photos to load despite infinite scroll.
+      loop do
+        execute_script('window.scrollTo(0, document.body.scrollHeight);')
+        links = find_link_elems_with('/photo.php')
+        break if links.length - photo_links.length == 0
+        photo_links = links
+      end
+
+      # Remove the cover photo and profile photo.
+      photo_links.shift
+      photo_links.shift
+
+      if photo_links.length == 0
+        return
+      end
+
+      photo_links.first.click
+      photo_links.length.times do
+        download_photo
+        find('.snowliftPager.next').click
+      end
+
+      # Add directory name for album.
+      title = find('.fbPhotoAlbumTitle').text.downcase.gsub(' ', '_')
+
+      begin
+        File.rename("#{Dir.pwd}/tmp/downloads", "#{Dir.pwd}/tmp/#{title}")
+
+      # This will fail if no files were downloaded because the `tmp/downloads`
+      # directory will not exist.
+      rescue Errno::ENOENT
+      end
+    end
+
+    def scrape_photos(username)
+      visit("/#{username}/photos_albums")
+      find_links_with('/media/set').each { |link| scrape_album(link) }
+    end
+
+    def login
+      visit('/login')
+      fill_in('email', with: ENV['FACEBOOK_EMAIL'])
+      fill_in('pass',  with: ENV['FACEBOOK_PASSWORD'])
+      click_button('loginbutton')
+
+      while has_css?('#approvals_code')
+        print 'Enter your 6-digit login code: '
+        fill_in('approvals_code', with: gets.chomp)
+        click_button('checkpointSubmitButton')
+      end
+
+      while has_css?('#checkpointSubmitButton')
+        click_button('checkpointSubmitButton')
+      end
+    end
+  end
+end

data/lib/facebook_profile_scraper/version.rb

@@ -0,0 +1,3 @@
+module FacebookProfileScraper
+  VERSION = '0.0.1'
+end

data/spec/scraper_spec.rb

@@ -0,0 +1,8 @@
+require 'facebook_profile_scraper'
+
+describe FacebookProfileScraper::Scraper, js: true do
+  it 'logs in' do
+    # subject.login
+    # expect(subject.page.find('#pagelet_welcome_box')).to be_true
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,110 @@
+require 'capybara'
+require 'capybara/poltergeist'
+
+Capybara.configure do |config|
+  config.default_driver = :poltergeist
+end
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: facebook_profile_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Maros Hluska
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-08-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.53'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.53'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+description: Scrape photos and other data from a friend's Facebook profile page
+email:
+- mhluska@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- facebook_profile_scraper.gemspec
+- lib/facebook_profile_scraper.rb
+- lib/facebook_profile_scraper/scraper.rb
+- lib/facebook_profile_scraper/version.rb
+- spec/scraper_spec.rb
+- spec/spec_helper.rb
+homepage: http://mhluska.com/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.6
+signing_key: 
+specification_version: 4
+summary: Scrape a friend's Facebook profile page
+test_files:
+- spec/scraper_spec.rb
+- spec/spec_helper.rb