ig_scrape 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a2d0a888ba09af0188257b8924cbaa83f85b15b0
+  data.tar.gz: 3d17f54b10d168c4aedc8eeadb40c280ffbb3fbc
+SHA512:
+  metadata.gz: 74804b533c38dfd21e4b49aa63380ce652f91700873f2a4916c6a3e591194d5027aa56520d9b7194c756be5016650e70234084537e239a1d76a166eb0c5f7cf3
+  data.tar.gz: 624898bb60e1a872e409d843947e214cb0c2691cc8fb6f1d3408d883dcdf9f5c61d86c2faf87182b9120e6cefc170b15442e6d60afa8b76fc485aef177795593

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.15.4

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in ig_scrape.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Trevor Kimenye
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# IG_Scrape
+
+This gem provides a utility for scraping instagram posts and comments
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+  gem 'ig_scrape'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ig_scrape
+
+## Usage
+
+### Getting information about an Instagram account
+
+  ```ruby
+    require 'ig_scrape'
+
+    client = IGScrape::Client.new("username")
+    puts client.follower_count
+    puts client.follows_count
+    puts client.post_count
+  ```
+
+### Loading all the posts for an instagram account  
+
+  ```ruby
+    require 'ig_scrape'
+    client = IGScrape::Client.new("username")
+    client.load
+
+    puts client.posts.length == client.post_count
+  ```
+
+### Loading a post
+
+  ```ruby
+    require 'ig_scrape'
+    code = "KVHudYDs"
+    post = IGScrape::Post.load_from_shortcode(code)
+    puts post.comment_count
+    puts post.has_more_comments?
+
+    post.load_more_comments
+    puts post.has_more_comments?
+  ```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ongair/ig_scrape.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ig_scrape"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ig_scrape.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "ig_scrape/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "ig_scrape"
+  spec.version       = IGScrape::VERSION
+  spec.authors       = ["Trevor Kimenye"]
+  spec.email         = ["kimenye@gmail.com"]
+
+  spec.summary       = %q{A gem to scrape instagram posts and comments}
+  spec.description   = %q{A gem to scrape instagram posts and comments.}
+  spec.homepage      = "https://github.com/ongair/ig_scrape"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "mocha"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "minitest-reporters"
+  spec.add_dependency "httparty"
+end

data/lib/ig_scrape/client.rb

@@ -0,0 +1,68 @@
+require 'httparty'
+
+class IGScrape::Client
+
+  attr_accessor :username, :full_name, :follower_count, :follows_count, :id ,:post_count, :profile_pic_url, :posts
+
+  def initialize(username)
+    @username = username
+    @posts = []
+    load_profile
+  end
+
+  def load
+    while has_more_posts? do
+      load_more_posts
+    end
+  end
+
+  def has_more_posts?
+    @posts.length < @post_count
+  end
+
+  def loaded_post_count
+    @posts.length
+  end
+
+  private
+
+    def load_profile
+      url = "https://www.instagram.com/#{@username}/?__a=1"
+      resp = HTTParty.get(url)
+
+      response = JSON.parse(resp.body)
+      user = response["user"]
+      @full_name = user["full_name"]
+      @follower_count = user["followed_by"]["count"]
+      @follows_count = user["follows"]["count"]
+      @id = user["id"]
+      @post_count = user["media"]["count"]
+      @page_info = user["media"]["page_info"]
+      @profile_pic_url = user["profile_pic_url"]
+
+      media = user["media"]["nodes"]
+      if media
+        @posts = media.collect do |node|
+          IGScrape::Post.new(node)
+        end
+      end
+    end
+
+    def load_more_posts
+      cursor = @page_info["end_cursor"]
+
+      variables = URI.encode_www_form_component("{\"id\":\"#{@id}\",\"first\":12,\"after\":\"#{cursor}\"}")
+      url = "https://www.instagram.com/graphql/query/?query_id=17888483320059182&variables=#{variables}"
+
+      resp = HTTParty.get(url)
+      response = JSON.parse(resp.body)
+      timeline = response["data"]["user"]["edge_owner_to_timeline_media"]
+      @page_info = timeline["page_info"]
+      new_posts = timeline["edges"].collect do |edge|
+        IGScrape::Post.new(IGScrape::Post.edge_timeline_to_payload(edge["node"]))
+      end
+
+      @posts = @posts.concat(new_posts)
+    end
+
+end

data/lib/ig_scrape/comment.rb

@@ -0,0 +1,19 @@
+class IGScrape::Comment
+
+  attr_accessor :id, :text, :created_at, :author_id, :author_name, :author_profile_pic
+
+  def initialize(payload)
+    load_from_payload(payload)
+  end
+
+  private
+
+    def load_from_payload payload
+      @id = payload["id"]
+      @text = payload["text"]
+      @created_at = payload["created_at"]
+      @author_id = payload["owner"]["id"]
+      @author_profile_pic = payload["owner"]["profile_pic_url"]
+      @author_name = payload["owner"]["username"]
+    end
+end

data/lib/ig_scrape/post.rb

@@ -0,0 +1,81 @@
+class IGScrape::Post
+
+  attr_accessor :id, :comment_count, :likes, :is_video, :code, :display_src, :caption, :created_at, :type, :comments
+
+  def initialize payload
+    @comments = []
+    load_from_payload(payload)
+  end
+
+  def self.load_from_shortcode code
+    url = "https://www.instagram.com/p/#{code}/?__a=1"
+    resp = HTTParty.get(url)
+    response = JSON.parse(resp.body)
+    payload = response["graphql"]["shortcode_media"]
+
+    post = IGScrape::Post.new(self.edge_timeline_to_payload(payload))
+  end
+
+  def has_more_comments?
+    @comments.length < @comment_count
+  end
+
+  def load_comments
+    while has_more_comments? do
+      load_more_comments
+    end
+  end
+
+  def self.edge_timeline_to_payload node
+    {
+      "id" => node["id"],
+      "__typename" => node["__typename"],
+      "is_video" => node["is_video"],
+      "code" => node["shortcode"],
+      "display_src" => node["display_url"],
+      "caption" => (node["edge_media_to_caption"]["edges"].length > 0 ? node["edge_media_to_caption"]["edges"].first["node"]["text"] : ""),
+      "date" => node["taken_at_timestamp"],
+      "comments" => node["edge_media_to_comment"],
+      "likes" => node["edge_media_preview_like"]
+    }
+  end
+
+  private
+
+    def load_more_comments
+      cursor = @comment_page_info["end_cursor"]
+      variables = URI.encode_www_form_component("{\"shortcode\":\"#{@code}\",\"first\":20,\"after\":\"#{cursor}\"}")
+
+      url = "https://www.instagram.com/graphql/query/?query_id=17852405266163336&variables=#{variables}"
+      resp = HTTParty.get(url)
+      response = JSON.parse(resp.body)
+
+      edges = response["data"]["shortcode_media"]["edge_media_to_comment"]["edges"]
+      new_comments = edges.collect do |edge|
+        IGScrape::Comment.new(edge["node"])
+      end
+
+      @comment_page_info = response["data"]["shortcode_media"]["edge_media_to_comment"]["page_info"]
+      @comments = @comments.concat(new_comments)
+    end
+
+    def load_from_payload payload
+      @id = payload["id"]
+      @is_video = payload["is_video"]
+      @type = payload["__typename"]
+      @caption = payload["caption"]
+      @created_at = payload["date"]
+      @display_src = payload["display_src"]
+      @code = payload["code"]
+      @likes = payload["likes"]["count"]
+      @comment_count = payload["comments"]["count"]
+      @comment_page_info = payload["comments"]["page_info"]
+
+      # load comments
+      if payload["comments"]["edges"]
+        @comments = payload["comments"]["edges"].collect do |edge|
+          IGScrape::Comment.new(edge["node"])
+        end
+      end
+    end
+end

data/lib/ig_scrape/version.rb

@@ -0,0 +1,3 @@
+module IGScrape
+  VERSION = "0.0.2"
+end

data/lib/ig_scrape.rb

@@ -0,0 +1,4 @@
+require "ig_scrape/version"
+require "ig_scrape/client"
+require "ig_scrape/post"
+require "ig_scrape/comment"

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: ig_scrape
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Trevor Kimenye
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-09-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A gem to scrape instagram posts and comments.
+email:
+- kimenye@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ig_scrape.gemspec
+- lib/ig_scrape.rb
+- lib/ig_scrape/client.rb
+- lib/ig_scrape/comment.rb
+- lib/ig_scrape/post.rb
+- lib/ig_scrape/version.rb
+homepage: https://github.com/ongair/ig_scrape
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: A gem to scrape instagram posts and comments
+test_files: []