insta_scraper 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4039c631a9e86f78c8fb7b25f46e63fd23945d45
-  data.tar.gz: 807caaab6fc4daa7a4aa8967b525d2a9c446e073
+  metadata.gz: c93649bb03ee2ae908a682a6e363dc49653bcc28
+  data.tar.gz: 7c9671c32fa26a221ec53753850087f7a967d872
 SHA512:
-  metadata.gz: b2ba575596682082d83e7c84549c0db54408007f7031ef469856b5988b8cc5315d4ac22247fba94c9840a057aa060efeb4ddc06f99e8ead3a5a3d606d5ec8795
-  data.tar.gz: 4f85446094701beb5ba1929adf6a3353d2f3d2aa1358331e6973224d0c9ca10c7678986777cd516d976940009b61293c6cbe326a2abbfb154f5590b8ca64796b
+  metadata.gz: 51a6b1f4c29c3769bca1d6adf0b7d3db0ad1377180b1039d4846e229e6b8305e51f7cbe0b1481e34905e54c00e6cd4ba0bc2ba18646557a671f25d01c90af368
+  data.tar.gz: 2d4c24a9967d5e03654013615b27aa0a864c5350883479ab1414ca4884a6ec00557bb3f6de45f1ab81f1ea9902ee125f1aaec68b1ffeaf22af6a3a90faa81016

data/README.md

@@ -70,14 +70,24 @@ account_media.data #=> #<Hashie::Mash...
 account_media.data.fetch('items') #=> [...]
 ```
 
-* InstaScraper::JSON::MediaComment
+* InstaScraper::JSON::MediaComment (deprecated, use MediaCommentStream)
 
 ```ruby
 media_comment = InstaScraper::JSON::MediaComment.new('BLeceL9BZNT', 17843235427151917, 20)
 # shortcode, last_comment_id, per_page
 
 media_comment.data #=> #<Hashie::Mash...
-media_comment.data['comments']['nodes'] #=> [#<Hashie::Mash created_at=1476385792.0 id="17843206063151917" text="Now we can get the golden penis ship!"
+media_comment.data['comments']['nodes'] #=> [#<Hashie::Mash created_at=1476385792.0 id="17843206063151917" text="Now we can get the golden ship!"
+```
+
+* InstaScraper::JSON::MediaCommentStream
+
+```ruby
+media_comments = InstaScraper::JSON::MediaCommentStream.new('BLeceL9BZNT', options = {})
+# optional_hash = { last_comment_id: '17845167841178842', max_comments: 200, per_page: 20}
+
+media_comments.data #=> #<Hashie::Mash...
+media_comments.data['comments'] #=> [#<Hashie::Mash created_at=1476385792.0 id="17843206063151917" text="Now we can get the golden ship!"
 ```
 
 

data/insta_scraper.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'hashie', '3.4.4'
+  spec.add_dependency 'faraday', '0.9.2'
+  spec.add_dependency 'faraday-cookie_jar', '0.0.6'
 
   spec.add_development_dependency 'bundler', '~> 1.12'
   spec.add_development_dependency 'rake', '~> 10.0'

data/lib/insta_scraper/html/account.rb

@@ -14,12 +14,5 @@ module InstaScraper
     def url
       "https://www.instagram.com/#{username}/#{serialize_params}"
     end
-
-    def data
-      @data ||=
-        Hashie::Mash.new(::JSON.parse(shared_data))
-                    .extend(Hashie::Extensions::DeepFetch)
-                    .extend(Hashie::Extensions::DeepFind)
-    end
   end
 end

data/lib/insta_scraper/html/media.rb

@@ -14,12 +14,5 @@ module InstaScraper
     def url
       "https://www.instagram.com/p/#{code}/#{serialize_params}"
     end
-
-    def data
-      @data ||=
-        Hashie::Mash.new(::JSON.parse(shared_data))
-                    .extend(Hashie::Extensions::DeepFetch)
-                    .extend(Hashie::Extensions::DeepFind)
-    end
   end
 end

data/lib/insta_scraper/html.rb

@@ -4,6 +4,13 @@ module InstaScraper
       @html ||= get_html
     end
 
+    def data
+      @data ||=
+        Hashie::Mash.new(::JSON.parse(shared_data))
+                    .extend(Hashie::Extensions::DeepFetch)
+                    .extend(Hashie::Extensions::DeepFind)
+    end
+
     protected
 
     def serialize_params

data/lib/insta_scraper/json/account.rb

@@ -9,12 +9,5 @@ module InstaScraper
     def url
       "https://www.instagram.com/#{username}/?__a=1"
     end
-
-    def data
-      @data ||=
-        Hashie::Mash.new(::JSON.parse(raw_json))
-                    .extend(Hashie::Extensions::DeepFetch)
-                    .extend(Hashie::Extensions::DeepFind)
-    end
   end
 end

data/lib/insta_scraper/json/account_media.rb

@@ -11,12 +11,5 @@ module InstaScraper
     def url
       "https://www.instagram.com/#{username}/media/#{serialize_params}"
     end
-
-    def data
-      @data ||=
-        Hashie::Mash.new(::JSON.parse(raw_json))
-                    .extend(Hashie::Extensions::DeepFetch)
-                    .extend(Hashie::Extensions::DeepFind)
-    end
   end
 end

data/lib/insta_scraper/json/media.rb

@@ -9,12 +9,5 @@ module InstaScraper
     def url
       "https://www.instagram.com/p/#{code}/?__a=1"
     end
-
-    def data
-      @data ||=
-        Hashie::Mash.new(::JSON.parse(raw_json))
-                    .extend(Hashie::Extensions::DeepFetch)
-                    .extend(Hashie::Extensions::DeepFind)
-    end
   end
 end

data/lib/insta_scraper/json/media_comment.rb

@@ -5,6 +5,7 @@ module InstaScraper
                 :per_page
 
     def initialize(shortcode, last_comment_id, per_page = 20)
+      warn 'Instagram changed things, this no longer works, use MediaCommentStream'
       @shortcode       = shortcode
       @last_comment_id = last_comment_id
       @per_page        = per_page
@@ -13,12 +14,5 @@ module InstaScraper
     def url
       "https://www.instagram.com/query/?q=ig_shortcode(#{shortcode}){comments.before(#{last_comment_id},#{per_page}){count,nodes{id,created_at,text,user{id,profile_pic_url,username,follows{count},followed_by{count},biography,full_name,media{count},is_private,external_url,is_verified}},page_info}}"
     end
-
-    def data
-      @data ||=
-        Hashie::Mash.new(::JSON.parse(raw_json))
-                    .extend(Hashie::Extensions::DeepFetch)
-                    .extend(Hashie::Extensions::DeepFind)
-    end
   end
 end

data/lib/insta_scraper/json/media_comment_stream.rb

@@ -0,0 +1,91 @@
+module InstaScraper
+
+  class JSON::MediaCommentStream < InstaScraper::JSON
+    attr_reader :shortcode,
+                :last_comment_id,
+                :per_page,
+                :max_comments,
+                :responses,
+                :comments,
+                :error
+
+    def initialize(shortcode, options = {})#, last_comment_id = nil, max_comments = nil, per_page = 20)
+      @shortcode       = shortcode
+      @last_comment_id = options[:last_comment_id] || nil
+      @per_page        = options[:per_page] || 20
+      @max_comments    = options[:max_comments] || nil
+      @responses       = {}
+      @comments        = {}
+    end
+
+    def data
+      Hashie::Mash.new({ comments: _data })
+                  .extend(Hashie::Extensions::DeepFetch)
+                  .extend(Hashie::Extensions::DeepFind)
+    end
+
+    private
+
+    def _data(current_last_comment_id = nil)
+      current_last_comment_id ||= (last_comment_id || default_last_comment_id)
+
+      response = connection.post do |req|
+        req.url '/query/'
+        #req.headers['Content-Type'] = 'application/json'
+        req.headers['content-type'] = 'application/x-www-form-urlencoded'
+        req.headers['X-CSRFToken'] = csrf_token
+        req.headers['Referer'] = "https://www.instagram.com/#{shortcode}/"
+        req.headers['x-instagram-ajax'] = '1'
+        req.headers['x-requested-with'] = 'XMLHttpRequest'
+        req.body = "q=ig_shortcode(#{shortcode}){comments.before(#{current_last_comment_id},#{per_page}){count,nodes{id,created_at,text,user{id,profile_pic_url,username,follows{count},followed_by{count},biography,full_name,media{count},is_private,external_url,is_verified}},page_info}}"
+      end
+
+      current_comments = ::JSON.parse(response.body)['comments']['nodes']
+
+      @responses[current_last_comment_id] = response
+      @comments[current_last_comment_id] = current_comments
+
+      if current_comments.any? && !reached_max_comments?
+        _data(current_comments.first['id'])
+      end
+
+      all_comments
+    rescue => e
+      warn e.inspect
+      @error = e
+      all_comments
+    end
+
+    def all_comments
+      comments.values.flatten
+    end
+
+    def reached_max_comments?
+      return false unless max_comments
+
+      all_comments.count > max_comments
+    end
+
+    def connection
+      @connection ||= Faraday.new(:url => "https://www.instagram.com/") do |builder|
+        builder.use :cookie_jar
+        builder.adapter Faraday.default_adapter
+      end
+    end
+
+    def connection_response
+      @connection_response ||= connection.get "/p/#{shortcode}/?__a=1"
+    end
+
+    def csrf_token
+      connection_response.headers["set-cookie"].match(%r{csrftoken=(?<token>\w+);})['token']
+    end
+
+    def default_last_comment_id
+      media = InstaScraper::JSON::Media.new(shortcode)
+      media.response = connection_response
+      media.data['media']['comments']['nodes'].last['id']
+    end
+  end
+
+end

data/lib/insta_scraper/json.rb

@@ -1,7 +1,20 @@
 module InstaScraper
   class JSON
     def raw_json
-      @raw_json ||= get_json
+      @raw_json ||= response.body
+    end
+
+    def data
+      @data ||=
+        Hashie::Mash.new(::JSON.parse(raw_json))
+                    .extend(Hashie::Extensions::DeepFetch)
+                    .extend(Hashie::Extensions::DeepFind)
+    end
+
+    attr_writer :response
+
+    def response
+      @response ||= Faraday.get(url)
     end
 
     protected
@@ -11,11 +24,5 @@ module InstaScraper
 
       "?" + params.map {|k, v| "#{k}=#{v}"}.join('&')
     end
-
-    private
-
-    def get_json
-      open(url).read
-    end
   end
 end

data/lib/insta_scraper/version.rb

@@ -1,3 +1,3 @@
 module InstaScraper
-  VERSION = '0.5.0'.freeze
+  VERSION = '0.6.0'.freeze
 end

data/lib/insta_scraper.rb

@@ -1,6 +1,8 @@
 require 'insta_scraper/version'
 
 require 'open-uri'
+require 'faraday'
+require 'faraday-cookie_jar'
 require 'json'
 
 require 'hashie'
@@ -14,3 +16,4 @@ require 'insta_scraper/json/account'
 require 'insta_scraper/json/media'
 require 'insta_scraper/json/account_media'
 require 'insta_scraper/json/media_comment'
+require 'insta_scraper/json/media_comment_stream'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: insta_scraper
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - preciz
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-14 00:00:00.000000000 Z
+date: 2016-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hashie
@@ -24,6 +24,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.4.4
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: faraday-cookie_jar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -90,6 +118,7 @@ files:
 - lib/insta_scraper/json/account_media.rb
 - lib/insta_scraper/json/media.rb
 - lib/insta_scraper/json/media_comment.rb
+- lib/insta_scraper/json/media_comment_stream.rb
 - lib/insta_scraper/version.rb
 homepage: 
 licenses: