nine-gag 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed51b9fff78efa65daf34555ba17d7a62d86a675
-  data.tar.gz: baab575f82d737a51c03a5e9465ff9c37206e5a1
+  metadata.gz: c6cfcfb89ee1e7d21c3d9a9fe9d94dd05fbb9c7d
+  data.tar.gz: 7256019e4e1d63ceb97586aa88881d0c5dd4bce4
 SHA512:
-  metadata.gz: f66eec53c4765f7b8d79ead0c9a100bb3f940c35a8586230f02d1641eea3cbfbc298d5676232f70bb6c43719f749d2c5ba58e9f5863211eb9ba8b4550fe516ae
-  data.tar.gz: 63a70b89bae89eaecec6f113e2d8ef4410945d43259abe7816943724495b17e233a64982d7c44d477530f6dff7054dd8435ecddab06bb6ef2160bee4013c8225
+  metadata.gz: cd2a140402c1043a61debcb2a9ea95cbf89e3846988c50e3a8c00437a1fa855cc6d508482c2172f263dd6436f62efb144432b1504c2d30326600904b2313f62e
+  data.tar.gz: 9d2f8dc9dc04d6357eac5892aa19afb96f48690a04232a43bba9e2b81b55e6fe55652d4bfa04d7a83b49606a6657aa684c26c7848068cf17d7cb793aaf828976

data/README.md

@@ -1,13 +1,13 @@
-| master | develop |
-| ------ | ------- |
-| [![Build Status](https://travis-ci.org/dimasjt/nine-gag.svg?branch=master)](https://travis-ci.org/dimasjt/nine-gag) | [![Build Status](https://travis-ci.org/dimasjt/nine-gag.svg?branch=develop)](https://travis-ci.org/dimasjt/nine-gag) |
+[![Gem Version](https://badge.fury.io/rb/nine-gag.svg)](https://badge.fury.io/rb/nine-gag)
+[![Build Status](https://travis-ci.org/dimasjt/nine-gag.svg?branch=develop)](https://travis-ci.org/dimasjt/nine-gag)
+[![Code Climate](https://codeclimate.com/github/dimasjt/nine-gag/badges/gpa.svg)](https://codeclimate.com/github/dimasjt/nine-gag)
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'nine-gag'
+gem 'nine-gag', '>= 0.1.3'
 ```
 
 And then execute:
@@ -20,30 +20,130 @@ Or install it yourself as:
 
 ## Usage
 
-### Get data from /section/type
+* [Posts](https://github.com/dimasjt/nine-gag/tree/develop#get-posts)
+* [Detail post](https://github.com/dimasjt/nine-gag/tree/develop#get-detail-post)
+* [Comments post](https://github.com/dimasjt/nine-gag/tree/develop#get-comments-post)
+
+### Get Posts
+#### Request
+```ruby
+# http://9gag.com/funny/hot
+NineGag.index('funny/hot')
+```
+
+#### Response
 ```ruby
-# http://9gag.com/gif/hot
-NineGag.index('gif/hot')
+[
+  {
+    :id => "ajqgWZp",
+    :title => "How deal with a Warlord",
+    :url => "http://9gag.com/gag/ajqgWZp",
+    :image => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460s.jpg",
+    :comments_count => 28,
+    :points => 464,
+    :media =>  {
+      :poster => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460s.jpg",
+      :mp4 => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460sv.mp4",
+      :webm => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460svwm.webm"
+    }
+  },
+  {
+    :id => "aERpNWM",
+    :title => "Some people really deserve this high-five",
+    :url => "http://9gag.com/gag/aERpNWM",
+    :image => "http://img-9gag-fun.9cache.com/photo/aERpNWM_460s.jpg",
+    :comments_count => 166,
+    :points => 6388,
+    :media => nil
+  },
+  ....
+]
 ```
 
 ### Load more data
 ```ruby
-NineGag.index('gif/hot', :last_id_post)
+NineGag.index('funny/hot', :last_id_post)
 
 # example
 # page 1
-posts = NineGag.index('gif/hot')
+posts = NineGag.index('funny/hot')
 
 # page 2
-last_id = posts.last.id
-posts = NineGag.index('gif/hot', last_id)
+last_id = posts.last[:id]
+posts = NineGag.index('funny/hot', last_id)
 ```
 
-### Detail post
+### Get Detail Post
+#### Request
 ```ruby
 post = NineGag.show(:post_id)
 ```
 
+#### Response
+```ruby
+{
+  :id => "ajqgWZp",
+  :title => "How deal with a Warlord",
+  :url => "http://9gag.com/gag/ajqgWZp",
+  :image => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460s.jpg",
+  :comments_count => 28,
+  :points => 464,
+  :media =>  {
+    :poster => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460s.jpg",
+    :mp4 => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460sv.mp4",
+    :webm => "http://img-9gag-fun.9cache.com/photo/ajqgWZp_460svwm.webm"
+  }
+}
+```
+
+### Get Comments Post
+#### Request
+```ruby
+NineGag.comments(:post_id)
+```
+
+#### Response
+```ruby
+[
+  {
+    :id => "c_148359676629369444",
+    :text => "http://i.memeful.com/media/post/YRO9Qqw_700wa_0.gif",
+    :timestamp => 1483596766,
+    :user_id => "u_13994024017199",
+    :permalink => "http://9gag.com/gag/a6Mg7mL#cs_comment_id=c_148359676629369444",
+    :points => 25,
+    :media => {
+      :jpg => "http://img-comment-fun.9cache.com/media/9a1d1430145033986894189858_700w_0.jpg",
+      :gif => "http://img-comment-fun.9cache.com/media/9a1d1430145033986894189858_700wa_0.gif",
+      :mp4 => "http://img-comment-fun.9cache.com/media/9a1d1430145033986894189858_700wv_0.mp4"
+    },
+    :user => {
+      :id => "u_13994024017199",
+      :avatar => "http://accounts-cdn.9gag.com/media/avatar/17248840_100_13.jpg",
+      :username=>"mister_widodo"
+    },
+    :children => [
+      {
+        :id => "c_148359764385634228",
+        :text => "@mister_widodo perfect",
+        :timestamp => 1483597643,
+        :user_id => "u_145203259185649618",
+        :permalink => "http://9gag.com/gag/a6Mg7mL#cs_comment_id=c_148359764385634228",
+        :points => 1,
+        :media => nil,
+        :user => {
+          :id => "u_145203259185649618",
+          :avatar => "http://accounts-cdn.9gag.com/media/avatar/27974937_100_1.jpg",
+          :username => "psoric39"
+        }
+      }
+    ],
+    :order_key=>"score_00000000001004_14835967662936"
+  },
+  .......
+]
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/dimasjt/nine-gag. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/lib/nine-gag.rb

@@ -5,19 +5,24 @@ require 'json'
 
 require 'nine-gag/version'
 require 'nine-gag/scraper'
+require 'nine-gag/generate'
 
 module NineGag
   def self.index(path, next_page = nil)
-    NineGag::Scraper.index(full_url(path), next_page)
+    data = NineGag::Scraper.new(path).index(next_page)
+
+    NineGag::Generate.new(data).index
   end
 
   def self.show(path)
-    NineGag::Scraper.show(full_url("gag/#{path}"))
+    data = NineGag::Scraper.new(path).show
+
+    NineGag::Generate.new(data).show
   end
 
-  private
+  def self.comments(id, next_page = nil)
+    data = NineGag::Scraper.new(id).comments(next_page)
 
-  def self.full_url(path)
-    "http://9gag.com/#{path}"
+    NineGag::Generate.new(data).comments
   end
 end

data/lib/nine-gag/generate.rb

@@ -0,0 +1,113 @@
+module NineGag
+  class Generate
+    def initialize(data)
+      @data = data
+    end
+
+    def index
+      @data = JSON.parse(@data.body)["items"]
+      @data.map do |post|
+        post = Nokogiri::HTML(post.last).search('article').first
+        generate_show_data(post, true)
+      end
+    end
+
+    def show
+      @data = @data.search('article').first
+      generate_show_data(@data)
+    end
+
+    def comments
+      @data = JSON.parse(@data.body)["payload"]["comments"]
+      @data.map do |comment|
+        generated_comment = generate_comment_data(comment)
+        generated_comment.merge!(
+          children: [],
+          order_key: comment["orderKey"]
+        )
+
+        unless comment["children"].empty?
+          comment["children"].each do |child|
+            generated_comment[:children].push generate_comment_data(child)
+          end
+        end
+
+        generated_comment
+      end
+    end
+
+    private
+
+    def generate_comment_data(comment)
+      {
+        id: comment["commentId"],
+        text: comment["text"],
+        timestamp: comment["timestamp"],
+        user_id: comment["userId"],
+        permalink: comment["permalink"],
+        points: comment["likeCount"],
+        media: media_comment_data(comment),
+        user: user_comment(comment["user"])
+      }
+    end
+
+    def user_comment(user)
+      {
+        id: user["userId"],
+        avatar: user["avatarUrl"],
+        username: user["displayName"]
+      }
+    end
+
+    def media_comment_data(comment)
+      return nil if comment["embedMediaMeta"]["embedImage"].nil?
+
+      {
+        jpg: media_comment(comment, "image"),
+        gif: media_comment(comment, "animated"),
+        mp4: media_comment(comment, "video")
+      }
+    end
+
+    def media_comment(comment, media)
+      comment["embedMediaMeta"].fetch("embedImage", {}).fetch(media, {}).fetch("url", nil)
+    end
+
+    def generate_show_data(post, index = false)
+      if index
+        title = post.search('h2.badge-item-title .badge-evt')
+      else
+        title = post.search('h2.badge-item-title')
+      end
+
+      post_meta = post.search('p.post-meta').first
+
+      {
+        id: post.attribute('data-entry-id').value,
+        title: title.text.strip,
+        url: post.attribute('data-entry-url').value,
+        image: image_data(post.search('div.badge-post-container a img').first),
+        comments_count: post_meta.search('a.comment').first.text.sub(' comments', '').sub(',', '').strip.to_i,
+        points: post_meta.search('.badge-item-love-count').first.text.sub(',', '').to_i,
+        media: media_data(post.search('video').first),
+        nsfw: !post.search('.nsfw-post').empty?
+      }
+    end
+
+    def image_data(image)
+      return "https://placeholdit.imgix.net/~text?txtsize=60&txt=NSFW&w=500&h=350&bg=000000&txtclr=ffffff" if image.nil?
+
+      image.attribute('src').value
+    end
+
+    def media_data(media)
+      return nil if media.nil?
+      videos = media.search('source')
+      {
+        poster: media.attribute('poster').value,
+        mp4: videos.first.attribute('src').value,
+        webm: videos.last.attribute('src').value
+      }
+    end
+  end
+end

data/lib/nine-gag/scraper.rb

@@ -1,96 +1,50 @@
 module NineGag
   class Scraper
-    # path = ":section/:type"
-    # next_page = "last_id"
-    def self.index(path, next_page = nil)
-      data = if next_page.nil?
-        scrape_html(path)
-      else
-        generate_json_posts(path, next_page)
-      end
-
-      generate_index_data(data, next_page)
+    def initialize(path)
+      @path = path
     end
 
-    # path = "gag/:id"
-    def self.show(path)
-      generate_show_data(scrape_html(path).search('article').first)
-    end
-
-    private
-
-    # will return Array of Nokogiri
-    def self.generate_json_posts(path, next_page)
-      items = JSON.parse(scrape_json(path, next_page).body)["items"]
-
-      items.map { |item| scrape_html(item.last).search('article').first }
-    end
-
-    def self.scrape_json(path, next_page)
-      RestClient.get(path,
-        { Accept: 'application/json', "X-Requested-With": 'XMLHttpRequest',
-          params: { id: next_page, c: 10 }
-        }
-      )
-    end
+    # path = ":section/:type"
+    # next_page = "last_id"
+    def index(next_page = nil)
+      url = full_url(@path)
 
-    # scrape html or from path
-    def self.scrape_html(path)
-      Nokogiri::HTML(path_or_html(path))
-    end
+      headers = {
+        Accept: 'application/json',
+        'X-Requested-With': 'XMLHttpRequest',
+        params: { id: next_page }
+      }
 
-    # check parameter if path or html
-    def self.path_or_html(data)
-      # if path
-      if data.length < 50
-        open data
-      else
-        data
-      end
+      RestClient.get(url, headers)
     end
 
-    def self.generate_show_data(post, index = false)
-      if index
-        title = post.search('h2.badge-item-title .badge-evt')
-      else
-        title = post.search('h2.badge-item-title')
-      end
-
-      post_meta = post.search('p.post-meta').first
-
-      {
-        id: post.attribute('data-entry-id').value,
-        title: title.text.strip,
-        url: post.attribute('data-entry-url').value,
-        image: image_data(post.search('div.badge-post-container a img').first),
-        comments_count: post_meta.search('a.comment').first.text.sub(' comments', '').sub(',', '').strip.to_i,
-        points: post_meta.search('a.point').first.text.sub(' points', '').sub(',', '').strip,
-        media: media_data(post.search('video').first)
+    # path = "gag/:id"
+    def show
+      url = full_url("gag/#{@path}")
+
+      Nokogiri::HTML(open(url))
+    end
+
+    def comments(next_page)
+      url = 'http://comment-cdn.9gag.com/v1/cacheable/comment-list.json'
+      params = {
+        appId: "a_dd8f2b7d304a10edaf6f29517ea0ca4100a43d1b",
+        url: full_url("gag/#{@path}"),
+        count: 10,
+        level: 2,
+        order: 'score',
+        mentionMapping: true,
+        origin: '9gag.com'
       }
-    end
-
-    def self.generate_index_data(scrape, next_page)
-      data = next_page.nil? ? scrape.search('article') : scrape
+      params.merge(ref: next_page) unless next_page.nil?
 
-      data.map do |post|
-        generate_show_data(post)
-      end
+      RestClient.get(url, { params: params })
     end
 
-    def self.image_data(image)
-      return nil if image.nil?
-
-      image.attribute('src').value
-    end
+    private
 
-    def self.media_data(media)
-      return nil if media.nil?
-      videos = media.search('source')
-      {
-        poster: media.attribute('poster').value,
-        mp4: videos.first.attribute('src').value,
-        webm: videos.last.attribute('src').value
-      }
+    def full_url(path)
+      "http://9gag.com/#{path}"
     end
   end
 end

data/lib/nine-gag/version.rb

@@ -1,3 +1,3 @@
 module NineGag
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nine-gag
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Dimas J. Taniawan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-23 00:00:00.000000000 Z
+date: 2017-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -149,6 +149,7 @@ files:
 - bin/console
 - bin/setup
 - lib/nine-gag.rb
+- lib/nine-gag/generate.rb
 - lib/nine-gag/scraper.rb
 - lib/nine-gag/version.rb
 - nine-gag.gemspec