qzone 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: de40edfabb8ee78b618d131b72e16b68b17d90e8
+  data.tar.gz: 0f942d1ecce66437adfc072755b77a5636b49573
+SHA512:
+  metadata.gz: cd036b848e27cd0024459529ea2e85d75332963e5bd1d9a7726be58e3ffffbdace1be3f8d4bbf8b0bceb9243a0ab19398af99c5d246da9d1a3bfebdd25f867f2
+  data.tar.gz: 8061cdf3f9c10af5001a56335c3f83e47ca7bf75c301961d3032774aed70e7640282588a89509576f10a5bb6979778744aa5a30f25fba7f3cf9437e26401225f

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 tuitu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Qzone
+
+Qzone是一个为qq空间定制的爬虫,它提供qzone的登录以及相册,好友等数据接口.
+
+## Installation
+
+`gem install qzone`
+
+## Usage
+
+首先,通过qq号和密码实例化一个Qzone类的对象:
+```ruby
+qzone = Qzone.new user, password
+```
+
+Qzone类提供了以下接口:
+- `ablums`:该方法接受任意一个合法的QQ号为参数,返回的是该QQ号下所有的相册构成了数组.每一个相册都是一个哈希,它包含以下项
+```ruby
+ablum[:host] # 拥有该相册的qq
+ablum[:id] # 相册id
+ablum[:name] # 相册名称
+ablum[:total] # 照片数量
+ablum[:allowAccess] # 相册权限 只有权限为1时, 该相册才是公开可爬的
+```
+- `photos_in_ablum`:该方法接受任意一个合法的相册哈希(由ablums接口返回的,或是手动构造相同结构的哈希),返回该相册下所有的照片组成的数组, 每一个相片都是一个哈希, 它包含以下项
+```ruby
+photo[:id] # 照片id
+photo[:name] # 照片名称
+photo[:url] # 照片的url, 可通过该url下载该照片
+```
+- `friends`:该方法无参数, 返回该qzone下所有的好友组成的数组,每一个好友都是一个哈希,它包含以下项
+```ruby
+friend[:uin] = # 该好友的qq号
+friend[:name] = # 该好友对于空间拥有者的备注
+friend[:score] # 亲密度评分
+friend[:img] = # 该好友的头像的url
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "qzone"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/qzone.rb

@@ -0,0 +1,193 @@
+require "qzone/version"
+require 'qzone/qzone_login'
+require "qzone/interface"
+
+class Qzone
+  def initialize(user, password)
+    @user = user
+    @password = password
+    @spider = Mechanize.new
+
+    if File.exist? "cookie#{@user}"
+      @spider.cookie_jar.load "cookie#{@user}"
+      skey = @spider.cookie_jar.find do |e|
+        e.name == 'p_skey'
+      end
+      @gtk = self.gtk(skey)
+
+      # if cookie is not valid
+      begin
+        unless self.cookies_valid?
+          File.delete "cookie#{@user}"
+          self.cookies
+        end
+      rescue
+        self.cookies
+      end
+
+    else
+      self.cookies
+    end
+  end
+
+  def cookies
+    account = {}
+    account[:user] = @user
+    account[:password] = @password
+
+    qzone = Login.new @user, @password, 5
+    qzone.login
+    @cookies = qzone.cookies
+    qzone.close
+
+    # construct cookies
+    @cookies.each do |cookie|
+      cookie[:expires] = Date.today + 1 if cookie[:expires].nil?
+      cookie = Mechanize::Cookie.new(
+        domain: cookie[:domain],
+        name: cookie[:name],
+        value: cookie[:value],
+        path: cookie[:path],
+        expires: cookie[:expires].to_s
+      )
+      @spider.cookie_jar << cookie
+    end
+
+    skey = @spider.cookie_jar.find do |e|
+      e.name == 'p_skey'
+    end
+
+    @gtk = self.gtk skey
+
+    @spider.request_headers = {
+      'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
+    }
+
+    @spider.cookie_jar.save_as "cookie#{@user}", session: true
+  end
+
+  def ablums(dest_qq)
+    result = @spider.get(sprintf(Interface["ablum"], @gtk.to_s, dest_qq, @user))
+
+    result = self.json result.content
+
+    ablums = []
+
+    return ablums if result['data']['albumListModeSort'].nil?
+
+    result['data']['albumListModeSort'].each do |e|
+      temp = {}
+      temp[:host] = dest_qq
+      temp[:id] = e['id']
+      temp[:name] = e['name']
+      temp[:total] = e['total']
+      temp[:allowAccess] = e['allowAccess'].to_i
+      # temp[:allowAccess] = 0 if temp[:allowAccess] != 1
+      ablums.push temp
+    end
+
+    ablums
+  end
+
+  def photos_in_ablum(ablum)
+    if ablum[:allowAccess] != 1
+      raise "can't access to ablum \"#{ablum[:name]}\""
+    end
+
+    photos = []
+
+    return photos if ablum[:total].to_i.zero?
+
+    total_photos = ablum[:total].to_i
+    pages = total_photos / 30
+    pages += 1 if total_photos % 30 != 0
+
+    pages.times do |e|
+      result = @spider.get(sprintf(Interface["photo"], @gtk, ablum[:host], ablum[:id], @user, (e*30).to_s))
+
+      result = self.json result.content
+
+      return photos if result['data']['photoList'].nil?
+
+      result['data']['photoList'].each do |e|
+        photo = {}
+        photo[:id] = e['id']
+        photo[:name] = e['name']
+        photo[:url] = e['url']
+
+        photos.push photo
+      end
+    end
+
+    photos
+  end
+
+  def friends
+    result = @spider.get(sprintf(Interface["friend"], @user, @gtk))
+
+    result = self.json result.content
+
+    friends = []
+
+    return friends if result['data']['items_list'].nil?
+
+    result['data']['items_list'].each do |e|
+      friend = {}
+      friend[:uin] = e['uin']
+      friend[:name] = e['name']
+      # 亲密度
+      friend[:score] = e['score']
+      friend[:img] = e['img']
+
+      friends.push friend
+    end
+
+    friends
+  end
+
+  def download_photos(photos, path)
+    worker = Mechanize.new
+    worker.request_headers = {
+      'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
+      "Referer" => "http://qzone.qq.com/"
+    }
+    photos.each do |e|
+      result = worker.get e[:url]
+      result.save_as path + e[:name] + "_#{Time.now.hash}" + '.jpg'
+    end
+  end
+
+
+  def gtk(skey)
+    raise "Skey can't be nil" if skey.nil?
+
+    hashes = 5381
+    skey.value.each_char do |c|
+      hashes += (hashes << 5) + c.ord
+    end
+    hashes & 0x7fffffff
+  end
+  # private
+
+  def json(str)
+    # str.gsub!(/shine0_Callback\(/, "" )
+    str.sub!(/.*_Callback\(/, '')
+    str.sub!(/\);/, '')
+
+    JSON.parse str
+  end
+
+  def cookies_valid?
+    begin
+      result = @spider.get(sprintf(Interface["ablum"], @gtk.to_s, @user, @user))
+    rescue Exception => error
+      if error.response_code == '403'
+        raise 'Connection refused'
+      else
+        raise 'unknown error happened in checking cookies'
+      end
+    end
+
+    !result.content.toutf8.include? '尚未登录或者登录超时'
+  end
+end

data/lib/qzone/interface.rb

@@ -0,0 +1,7 @@
+class Qzone
+  Interface = {
+    "ablum" => "http://h5.qzone.qq.com/proxy/domain/tjalist.photo.qq.com/fcgi-bin/fcg_list_album_v3?g_tk=%s&callback=shine0_Callback&t=419043014&hostUin=%s&uin=%s&appid=4&inCharset=utf-8&outCharset=utf-8&source=qzone&plat=qzone&format=jsonp&notice=0&filter=1&handset=4&pageNumModeSort=40&pageNumModeClass=15&needUserInfo=1&idcNum=5&callbackFun=shine0&_=1475649079634",
+    "photo" => "http://h5.qzone.qq.com/proxy/domain/tjplist.photo.qzone.qq.com/fcgi-bin/cgi_list_photo?g_tk=%s&callback=shine0_Callback&t=775643811&mode=0&idcNum=5&hostUin=%s&topicId=%s&noTopic=0&uin=%s&pageStart=%s&pageNum=30&skipCmtCount=0&singleurl=1&batchId=&notice=0&appid=4&inCharset=utf-8&outCharset=utf-8&source=qzone&plat=qzone&outstyle=json&format=jsonp&json_esc=1&question=&answer=&callbackFun=shine0&_=1475649257165",
+    "friend" => "https://h5.qzone.qq.com/proxy/domain/r.qzone.qq.com/cgi-bin/tfriend/friend_ship_manager.cgi?uin=%s&do=1&rd=0.9983570276719183&fupdate=1&clean=1&g_tk=%s"
+  }
+end

data/lib/qzone/qzone_login.rb

@@ -0,0 +1,99 @@
+require 'selenium-webdriver'
+require 'mechanize'
+require 'date'
+require 'yaml'
+require 'json'
+
+class Qzone
+  class Login
+    def initialize user, password, wait_time
+      @user = user
+      @password = password
+
+      begin
+        user_agent = 'User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'
+
+        capabilities = Selenium::WebDriver::Remote::Capabilities.phantomjs(
+        'phantomjs.page.settings.userAgent' => user_agent,
+        "phantomhs.page.settings.loadImages" => false
+        )
+        @driver = Selenium::WebDriver.for(:phantomjs, :desired_capabilities => capabilities)
+        @waiter = Selenium::WebDriver::Wait.new(timeout: wait_time)
+      rescue
+        raise 'init error,make sure phantomjs has been installed correctly.'
+      end
+    end
+
+    def login
+      @waiter.until {@driver.navigate.to 'http://qzone.qq.com/'}
+
+      @waiter.until {@driver.switch_to.frame 'login_frame'}
+
+      @waiter.until { @driver.find_element(id: 'switcher_plogin') }
+      plogin = @driver.find_element(id: 'switcher_plogin')
+      plogin.click
+
+      user = @driver.find_element(id: 'u')
+      user.send_keys @user
+      password = @driver.find_element(id: 'p')
+      password.send_keys @password
+
+      @waiter.until { @driver.find_element(xpath: '//*[@id="login_button"]') }
+      login = @driver.find_element(xpath: '//*[@id="login_button"]')
+
+      login.click
+
+      sleep(1)
+
+      # 如果正常的话 目前就已经返回true
+      return @pass = true if @driver.current_url != 'http://qzone.qq.com/'
+
+      # 下面分析可能的原因
+      begin
+        @waiter.until {@driver.find_element(id: 'vcode')}
+        raise 'Need verify code, please manualy login Qzone first'
+      rescue Exception => error
+        if error.message == 'Need verify code, please manualy login Qzone first'
+          raise error.message
+        end
+      end
+
+      # if find element with id "err_m", indicate something wrong
+      begin
+        @waiter.until {@driver.find_element(id: 'err_m')}
+        err = @driver.find_element(id: 'err_m')
+        error_message = err.text
+      rescue
+        # if can't find that element, indicate something wrong happened but ->
+        # we don't know about that.
+        raise 'Unknown error happened in login'
+      end
+
+      if error_message.include? '您输入的帐号或密码不正确'
+        raise 'Account error'
+      elsif error_message.include? '网络繁忙'
+        raise 'Network busy, try later'
+      else
+        raise 'Unknown error happened in login'
+      end
+    end
+
+    def gtk
+      # 获取gtk
+      @skey = @driver.manage.cookie_named 'p_skey'
+      hashes = 5381
+      @skey[:value].each_char do |c|
+        hashes += (hashes << 5) + c.ord
+      end
+      hashes & 0x7fffffff
+    end
+
+    def cookies
+      @driver.manage.all_cookies
+    end
+
+    def close
+      @driver.close if @driver
+    end
+  end
+end

data/lib/qzone/version.rb

@@ -0,0 +1,3 @@
+class Qzone
+  VERSION = "0.1.0"
+end

data/qzone.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'qzone/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "qzone"
+  spec.version       = Qzone::VERSION
+  spec.authors       = ["tuitu"]
+  spec.email         = ["1965972530@qq.com"]
+
+  spec.summary       = %q{"a spider aimed for qzone"}
+  spec.description   = %q{"provide apis to crawl data from qzone"}
+  spec.homepage      = "https://github.com/hellotuitu/qzone"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "selenium-webdriver"
+  spec.add_development_dependency "mechanize"
+  spec.add_development_dependency "json"
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: qzone
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- tuitu
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: '"provide apis to crawl data from qzone"'
+email:
+- 1965972530@qq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/qzone.rb
+- lib/qzone/interface.rb
+- lib/qzone/qzone_login.rb
+- lib/qzone/version.rb
+- qzone.gemspec
+homepage: https://github.com/hellotuitu/qzone
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: '"a spider aimed for qzone"'
+test_files: []