newrank 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eb1cbfb901aa3e8dc9117fc5df14b111596742ee
+  data.tar.gz: a95c42da710abe8ca57b132d9cc1365c1b393ef5
+SHA512:
+  metadata.gz: 3971adcf2000608d38b9edd52179fe53637ab57ff6d9684569caf643a2324c65ec127c5afa07e1bc2aacee14af9a54a25c0c06f4b4ba1b886f9c2b6519552354
+  data.tar.gz: 1bcdf263496f182b74c3c47c843882547ebe64f9e49f39fca2b2fe18fafc5b2778049558bed8f8d3460949c8eaa96c67022e6c3030e7c80afe5e98b38b4e2e75

data/lib/newrank.rb

@@ -0,0 +1,139 @@
+# coding: utf-8
+require 'open-uri'
+require 'rkelly'
+require 'nokogiri'
+require 'json'
+require 'v8'
+require 'execjs'
+require 'rest-client'
+
+class Newrank
+  # crawl newrank info
+  def crawl(newrank_id)
+    doc = document(newrank_id.gsub("\u{a0}",""))
+    if !doc.nil?
+      score, uuid = score_and_uuid(doc)
+
+      element = doc.css(".detail-fans-counts")[0]
+      active_users_count = element.nil? ? 0 : element.text.gsub(",","").to_i
+
+      element = doc.css(".info-detail-head-weixin-fun-introduce")[0]
+      introduce = element.nil? ? "" : element.text
+
+      week_data = week_data(doc)
+      if !uuid.nil?
+        posts_data = fetch_post(uuid)
+      end
+      {
+        active_users_count: active_users_count,
+        score: (score || 0),
+        introduce: introduce,
+        week_data: week_data,
+        posts_data: (posts_data || {})
+      }
+    else
+      {
+        active_users_count: 0,
+        score: 0,
+        introduce: "",
+        week_data: [],
+        posts_data: {}
+      }
+    end
+  end
+  
+  # crawl posts
+  def fetch_post(uuid)
+    nonce = gen_nonce
+		xyz = gen_xyz(nonce, uuid)
+
+    posts = JSON.parse(RestClient.post("http://www.newrank.cn/xdnphb/detail/getAccountArticle", {uuid: uuid, nonce: nonce, xyz: xyz}, {"User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"}))
+  end
+
+  # crawl week data
+  def week_data(doc)
+    data = []
+
+    if !doc.css("script")[0].nil?
+      parser = RKelly::Parser.new
+      ast = parser.parse(doc.css("script")[0].text.strip)
+      array_node = ast.pointcut(RKelly::Nodes::ArrayNode).matches.first
+      array_node.pointcut(RKelly::Nodes::ElementNode).matches.each do |element_node|
+        data << JSON.parse(element_node.to_ecma)
+      end
+    end
+
+    data
+  end
+  
+  # get Nogogiri Document
+  def document(newrank_account)
+    url = 'http://www.newrank.cn/public/info/detail.html?account=' + newrank_account
+    Nokogiri::HTML(open(url, "User-Agent" => "Mozilla/5.0 (Windows NT 6.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1", :read_timeout => 10), nil, 'utf-8')
+  end
+  
+  # find score and uuid
+  def score_and_uuid(doc)
+    score, uuid = nil
+
+    script = doc.css("script")[0]
+    if !script.nil?
+      parser = RKelly::Parser.new
+      ast = parser.parse(script.text.strip)
+      array_node = ast.pointcut(RKelly::Nodes::ArrayNode).matches.first
+      element_node = array_node.pointcut(RKelly::Nodes::ElementNode).matches.first
+      json_data = element_node.nil? ? {} : JSON.parse(element_node.to_ecma)
+      if json_data["new_rank_index_mark"]
+        score = json_data["new_rank_index_mark"].to_f
+      else
+        score = 0.0
+      end
+      object_node = ast.pointcut(RKelly::Nodes::AssignExprNode).matches[-1]
+      node = object_node.pointcut(RKelly::Nodes::PropertyNode).matches.select{|n| n.name == '"uuid"'}.first.value
+      uuid = node.value[1..-2]
+    end
+
+    return score, uuid
+  end
+  
+  # wait for seconds
+  # instead of request too much
+  def wait_for_seconds
+		sleep(1 * rand)
+	end
+  
+  # generate parameter nonce
+	def gen_nonce
+		a = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a","b", "c", "d", "e", "f"]
+		b = 0
+		while 500 > b
+			d = 0
+			c = ""
+			while 9 > d
+				e = (16 * rand).floor
+				c << a[e]
+				d = d + 1
+			end
+			b = b + 1
+		end
+    c
+	end
+  
+  # generate parameter xyz
+	def gen_xyz(nonce, uuid)
+    h = "/xdnphb/detail/getAccountArticle?AppKey=joker&uuid=#{uuid}&nonce=#{nonce}"
+	  _md5(h)
+	end
+  
+  # use js md5 algorightm, written by newrank, file in assets/newrank_md5.js
+  def _md5(str)
+    js_context.call('newrank_md5', str, bare: true)
+  end
+  
+  # js context
+  def js_context
+    file_path = File.join( File.dirname(__FILE__), './assets/newrank_md5.js')
+    @context ||= ExecJS.compile(File.read(file_path))
+  end
+  #------------------------
+end

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification
+name: newrank
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Tesla Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-10-25 00:00:00.000000000 Z
+dependencies: []
+description: A Crawler for NewRank
+email: leechee89@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/newrank.rb
+homepage: https://github.com/liqites/newrank_crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Newrank Crawler
+test_files: []