insta_scrape 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34460e7bb43c001ca79d3c2c3e4cd058d87cb407
-  data.tar.gz: 69e80191442e15dd7eaa6b3b23b06fd6c9874615
+  metadata.gz: abbaf8a5b61d0f1d25d3480a64b16eedacd32b58
+  data.tar.gz: a67b9a14e265d90f47dbd566b21d64e42694fe43
 SHA512:
-  metadata.gz: 58f25720adaa3fbf538f6aedd3c3dc2ad5d16975afb9b8287934ea08fac3006b9b22cd7dab0ac36d8bd820c2cfad3914714d09703559dd61d84f8c2c6cc527dd
-  data.tar.gz: ec7b603c8425cac9d446ad8dc8d6bada6acda4aa275cbcf85a526f9d0484aa167c8170c2de329d0ec83f8702cd4ddb996ee02e21fbaed9a6db7b7ca80712589b
+  metadata.gz: 5a2cf697cafb2e5a46ac4dad8453666cb50d4c2923dae7d6344b9fd35aef517798d4a45c8b0d260c7f780b1036505eafd6efaa716f3b0759892ed5b6f97fd1fd
+  data.tar.gz: 235eb28eee937dfa4350ef708b1117b96f49689d89a2c65f95bc3fcd3e4514f170cb491544c58507225ba71a1a7403c235bd1e72543e775119f8d2da19bb0ca0

data/README.md

@@ -5,9 +5,11 @@
 A ruby scraper for instagram in 2016. Because the hashtag deprecation in the API is just silly.
 This gem is dependent on Capybara, PhantomJS, and Poltergeist.
 
+Using this gem you can access multiple facets of the instagram API without needing authorization, most importantly the hashtag.
+
 ## Note
 
-The number of results may vary as this isn't an official endpoint.
+The number of results may vary when using certain methods as this isn't an official endpoint.
 
 ## Todo
 
@@ -32,41 +34,95 @@ Or install it yourself as:
 
 ## Usage
 
-The scrape maps the response objects to an array. The objects currently have 2 attributes, "link" and "image".
+###Available methods
+
+As of right now, each method accepts only one argument - a hashtag or a username.
+
+```ruby
+#scrape a hashtag for as many results as possible
+InstaScrape.hashtag("test")
+#scrape all user info
+InstaScrape.user_info("dannyvassallo")
+#scrape all user info and posts
+InstaScrape.user_info_and_posts('foofighters')
+#scrape just a users posts (as many as possible)
+InstaScrape.user_posts('foofighters')
+#scrape a users follower count
+InstaScrape.user_follower_count('foofighters')
+#scrape a users following count
+InstaScrape.user_following_count('foofighters')
+#scrape a users post count
+InstaScrape.user_post_count('foofighters')
+#scrape a users description
+InstaScrape.user_description('foofighters')
+```
 
-The simplest use is the following case:
+####Hashtag, User Post, and Nested Posts Scrape
 
 ```ruby
-#InstaScrape takes one argument. In this case its the #test hashtag.
-@insta_scrape = InstaScrape.new
-scrape_result = @insta_scrape.hashtag("test")
+#basic use case
+
+#scrape_result = InstaScrape.user_info_and_posts('foofighters').posts
+#scrape_result = InstaScrape.user_posts('foofighters')
+scrape_result = InstaScrape.hashtag("test")
 scrape_result.each do |post|
-  puts post["image"]
-  puts post["link"]
+  puts post.image
+  puts post.link
 end
 ```
 
 Here is a `.erb` example using MaterializeCSS to render the posts as cards:
-```ruby
 
+```ruby
 #in your controller or helper assuming you aren't storing the posts
-@insta_scrape = InstaScrape.new
-@posts = @insta_scrape.hashtag("test")
 
+#@posts = InstaScrape.user_info_and_posts('foofighters').posts
+#@posts = InstaScrape.user_posts('foofighters')
+@posts = InstaScrape.hashtag("test")
+```
+
+```ruby
 # your .erb file
-<% @posts.each do |post| %>
-<div class="col s12 m6 l4">
-  <div class="card hoverable">
-    <div class="card-image"><a href="<%= post['link'] %>"><img src="<%= post['image'] %>"></a></div>
-    <div class="card-content">
-      <!-- <p></p> -->
+# access post attributes using dot notation
+<div class="row">
+  <% @posts.each do |post| %>
+  <div class="col s12 m6 l4">
+    <div class="card hoverable">
+      <div class="card-image"><a href="<%= post.link %>"><img src="<%= post.image %>"></a></div>
+      <div class="card-content">
+        <!-- <p></p> -->
+      </div>
+      <div class="card-action center-align"><a class="btn black" href="<%= post.link %>">Open Post</a></div>
     </div>
-    <div class="card-action center-align"><a class="btn black" href="<%= post['link'] %>">Open Post</a></div>
   </div>
+  <% end %>
 </div>
-<% end %>
 ```
 
+####User Info
+
+All user information is accessible using dot notation.
+If we run:
+```ruby
+u = InstaScrape.user_info("foofighters")
+```
+We then have access to the following attributes:
+```ruby
+u.image
+#returns => "https://instagram.fewr1-2.fna.fbcdn.net/t51.2885-19/11856782_370180896524950_961003442_a.jpg"
+u.post_count
+#returns => "305"
+u.follower_count
+#returns  => "1.5m"
+u.following_count
+#returns => "35"
+```
+
+Each of these attributes is accessible using the methods listed above as well.
+
+Using `u = InstaScrape.user_info_and_posts('foofighters')` will give access to the `u.posts` attribute and can be iterated through.
+The example above covers this.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/insta_scrape.gemspec

@@ -22,12 +22,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "capybara", "~> 2.7.1"
-  spec.add_development_dependency "phantomjs", "~> 2.1.1.0"
-  spec.add_development_dependency "poltergeist", "~> 1.9.0"
+  spec.add_development_dependency "capybara", "~> 2.7.1", ">= 2.7.1"
+  spec.add_development_dependency "phantomjs", "~> 2.1.1.0", ">= 2.1.1.0"
+  spec.add_development_dependency "poltergeist", "~> 1.9.0", ">= 1.9.0"
 
-  spec.add_runtime_dependency "capybara", ">= 2.7.1"
-  spec.add_runtime_dependency "phantomjs", ">= 2.1.1.0"
-  spec.add_runtime_dependency "poltergeist", ">= 1.9.0"
+  spec.add_runtime_dependency "capybara", "~> 2.7.1", ">= 2.7.1"
+  spec.add_runtime_dependency "phantomjs", "~> 2.1.1.0", ">= 2.1.1.0"
+  spec.add_runtime_dependency "poltergeist", "~> 1.9.0", ">= 1.9.0"
 
 end

data/lib/dependencies.rb

@@ -0,0 +1,7 @@
+require "insta_scrape/version"
+Dir[File.dirname(__FILE__) + '/models/*.rb'].each {|file| require file }
+require "capybara"
+require "capybara/dsl"
+require "capybara/poltergeist"
+require "phantomjs"
+require "init/poltergeist"

data/lib/init/poltergeist.rb

@@ -0,0 +1,5 @@
+Capybara.register_driver :poltergeist do |app|
+    Capybara::Poltergeist::Driver.new(app, :phantomjs => Phantomjs.path)
+end
+
+Capybara.default_driver = :poltergeist

data/lib/insta_scrape/version.rb

@@ -1,3 +1,3 @@
-class InstaScrape
-  VERSION = "0.1.0"
+module InstaScrape
+  VERSION = "1.0.0"
 end

data/lib/insta_scrape.rb

@@ -1,22 +1,94 @@
-require "insta_scrape/version"
-require "capybara"
-require "capybara/dsl"
-require "capybara/poltergeist"
-require "phantomjs"
-
-Capybara.register_driver :poltergeist do |app|
-    Capybara::Poltergeist::Driver.new(app, :phantomjs => Phantomjs.path)
-end
-
-Capybara.default_driver = :poltergeist
+require "dependencies"
 
-class InstaScrape
-  include Capybara::DSL
+module InstaScrape
+  extend Capybara::DSL
 
-  def hashtag(hashtag)
+  #get a hashtag
+  def self.hashtag(hashtag)
     visit "https://www.instagram.com/explore/tags/#{hashtag}/"
     @posts = []
+    scrape_posts
+  end
+
+  #get user info
+  def self.user_info(username)
+    scrape_user_info(username)
+    @user = InstaScrape::InstagramUser.new(username, @image, @post_count, @follower_count, @following_count, @description)
+  end
+
+  #get user info and posts
+  def self.user_info_and_posts(username)
+    scrape_user_info(username)
+    scrape_user_posts(username)
+    @user = InstaScrape::InstagramUserWithPosts.new(username, @image, @post_count, @follower_count, @following_count, @description, @posts)
+  end
+
+  #get user posts only
+  def self.user_posts(username)
+    scrape_user_posts(username)
+  end
+
+  #get user follower count
+  def self.user_follower_count(username)
+    scrape_user_info(username)
+    return @follower_count
+  end
+
+  #get user following count
+  def self.user_following_count(username)
+    scrape_user_info(username)
+    return @following_count
+  end
+
+  #get user post count
+  def self.user_post_count(username)
+    scrape_user_info(username)
+    return @post_count
+  end
+
+  #get user description
+  def self.user_description(username)
+    scrape_user_info(username)
+    return @description
+  end
+
+  private
+  #post iteration method
+  def self.iterate_through_posts
+    all("article div div div a").each do |post|
+
+      link = post["href"]
+      image = post.find("img")["src"]
+      info = InstaScrape::InstagramPost.new(link, image)
+      @posts << info
+
+    end
 
+    #log
+    puts "POST COUNT: #{@posts.length}"
+    self.log_posts
+    #return result
+    return @posts
+  end
+
+  #user info scraper method
+  def self.scrape_user_info(username)
+    visit "https://www.instagram.com/#{username}/"
+    @image = page.find('article header div img')["src"]
+    within("header") do
+      post_count_html = page.find('span', :text => "posts", exact: true)['innerHTML']
+      @post_count = get_span_value(post_count_html)
+      follower_count_html = page.find('span', :text => "followers", exact: true)['innerHTML']
+      @follower_count = get_span_value(follower_count_html)
+      following_count_html = page.find('span', :text => "following", exact: true)['innerHTML']
+      @following_count = get_span_value(following_count_html)
+      description = page.find('h2').first(:xpath,".//..")['innerHTML']
+      @description = Nokogiri::HTML(description).text
+    end
+  end
+
+  #scrape posts
+  def self.scrape_posts
     begin
       page.find('a', :text => "Load more", exact: true).click
       max_iteration = 10
@@ -34,28 +106,27 @@ class InstaScrape
     end
   end
 
-  private
-
-  def iterate_through_posts
-    all("article div div div a").each do |post|
-
-      link = post["href"]
-      image = post.find("img")["src"]
-
-      info = {
-        "link" => link,
-        "image" => image
-      }
-
-      @posts << info
+  def self.scrape_user_posts(username)
+    @posts = []
+    visit "https://www.instagram.com/#{username}/"
+    scrape_posts
+  end
 
+  #post logger
+  def self.log_posts
+    @posts.each do |post|
+      puts "\n"
+      puts "Image: #{post.image}\n"
+      puts "Link: #{post.link}\n"
     end
+    puts "\n"
+  end
 
-    #log
-    puts "POST COUNT: #{@posts.length}"
-
-    #return result
-    return @posts
+  #split away span tags from user info numbers
+  def self.get_span_value(element)
+    begin_split = "\">"
+    end_split = "</span>"
+    return element[/#{begin_split}(.*?)#{end_split}/m, 1]
   end
 
 end

data/lib/models/instagram_post.rb

@@ -0,0 +1,7 @@
+class InstaScrape::InstagramPost
+  attr_accessor :link, :image
+  def initialize(link, image)
+    @image = image
+    @link = link
+  end
+end

data/lib/models/instagram_user.rb

@@ -0,0 +1,11 @@
+class InstaScrape::InstagramUser
+  attr_accessor :username, :image, :post_count, :follower_count, :following_count, :description
+  def initialize(username, image, post_count, follower_count, following_count, description)
+    @username = username
+    @image = image
+    @post_count = post_count
+    @follower_count = follower_count
+    @following_count = following_count
+    @description = description
+  end
+end

data/lib/models/instagram_user_with_posts.rb

@@ -0,0 +1,12 @@
+class InstaScrape::InstagramUserWithPosts
+  attr_accessor :username, :image, :post_count, :follower_count, :following_count, :description, :posts
+  def initialize(username, image, post_count, follower_count, following_count, description, posts)
+    @username = username
+    @image = image
+    @post_count = post_count
+    @follower_count = follower_count
+    @following_count = following_count
+    @description = description
+    @posts = posts
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: insta_scrape
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - dannyvassallo
@@ -59,6 +59,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.7.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -66,6 +69,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.7.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
 - !ruby/object:Gem::Dependency
   name: phantomjs
   requirement: !ruby/object:Gem::Requirement
@@ -73,6 +79,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -80,6 +89,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.1.0
 - !ruby/object:Gem::Dependency
   name: poltergeist
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +99,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.9.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -94,10 +109,16 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.9.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.1
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.7.1
@@ -105,6 +126,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.1
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.7.1
@@ -112,6 +136,9 @@ dependencies:
   name: phantomjs
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.1.0
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.1.0
@@ -119,6 +146,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.1.0
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.1.0
@@ -126,6 +156,9 @@ dependencies:
   name: poltergeist
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.0
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.0
@@ -133,6 +166,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.0
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.0
@@ -155,8 +191,13 @@ files:
 - bin/console
 - bin/setup
 - insta_scrape.gemspec
+- lib/dependencies.rb
+- lib/init/poltergeist.rb
 - lib/insta_scrape.rb
 - lib/insta_scrape/version.rb
+- lib/models/instagram_post.rb
+- lib/models/instagram_user.rb
+- lib/models/instagram_user_with_posts.rb
 homepage: https://github.com/dannyvassallo/insta_scrape
 licenses:
 - MIT