weget 0.1.0 → 0.1.1

data/lib/weget/config.rb

@@ -0,0 +1,28 @@
+# coding: utf-8
+
+USER_ID = "xxxxxx"
+USER_PASS = "xxxxxx"
+
+URL_SERVICE = 'https://www.iauc.co.jp/service/'
+URL_TAB_CREATE = 'https://www.iauc.co.jp/ajax/tab/create'
+URL_NOTICE_PRE_LOGIN = 'https://www.iauc.co.jp/ajax/service/notice_pre_login'
+URL_REQUIRED_NOTICE = 'https://www.iauc.co.jp/ajax/service/required_notice'
+URL_REQUIRED_NOTICE_TOP = 'https://www.iauc.co.jp/ajax/service/required_notice_top'
+URL_LOGIN = 'https://www.iauc.co.jp/service/login'
+URL_MARKET = 'https://www.iauc.co.jp/market/'
+URL_LOGOUT = 'https://www.iauc.co.jp/service/logout'
+
+URL_TAB_CHANGE = 'https://www.iauc.co.jp/ajax/tab/change'
+URL_SEARCH_MARKET = 'https://www.iauc.co.jp/ajax/search/makers?vehicleType=car&mode=market&siteId=25%2C12%2C14%2C13%2C15%2C11%2C10%2C65%2C231%2C2%2C4%2C1%2C20%2C22%2C23%2C21%2C163%2C28%2C253%2C62%2C63%2C68%2C71%2C80%2C51%2C58%2C117%2C93%2C250%2C142%2C148%2C139%2C146%2C147%2C192%2C193%2C211%2C216%2C219%2C197%2C198%2C217%2C212%2C3%2C89%2C165%2C40%2C91%2C43%2C114%2C111%2C252%2C224%2C7%2C90%2C59%2C57%2C84%2C52%2C81%2C140%2C194%2C18%2C19%2C16%2C143%2C39%2C42%2C6%2C29%2C35%2C37%2C70%2C79%2C54%2C61%2C60%2C67%2C87%2C95%2C88%2C119%2C251%2C196%2C195%2C240%2C241%2C242%2C86%2C55%2C64%2C53%2C66%2C82%2C48%2C221%2C222%2C223%2C30%2C73%2C33%2C41%2C113%2C237%2C238%2C239%2C243%2C5%2C74%2C78%2C56%2C206%2C31%2C235%2C17%2C34%2C36%2C38&_=1524808327595'
+URL_SEARCH_MARKET2 = 'https://www.iauc.co.jp/market/search?d=25%2C12%2C14%2C13%2C15%2C11%2C10%2C65%2C231%2C2%2C4%2C1%2C20%2C22%2C23%2C21%2C163%2C28%2C253%2C62%2C63%2C68%2C71%2C80%2C51%2C58%2C117%2C93%2C250%2C142%2C148%2C139%2C146%2C147%2C192%2C193%2C211%2C216%2C219%2C197%2C198%2C217%2C212%2C3%2C89%2C165%2C40%2C91%2C43%2C114%2C111%2C252%2C224%2C7%2C90%2C59%2C57%2C84%2C52%2C81%2C140%2C194%2C18%2C19%2C16%2C143%2C39%2C42%2C6%2C29%2C35%2C37%2C70%2C79%2C54%2C61%2C60%2C67%2C87%2C95%2C88%2C119%2C251%2C196%2C195%2C240%2C241%2C242%2C86%2C55%2C64%2C53%2C66%2C82%2C48%2C221%2C222%2C223%2C30%2C73%2C33%2C41%2C113%2C237%2C238%2C239%2C243%2C5%2C74%2C78%2C56%2C206%2C31%2C235%2C17%2C34%2C36%2C38&referer_site=true'
+URL_SEARCH_VEHICLE = 'https://www.iauc.co.jp/ajax/search/vehicle_names?vehicleType=car&makerId=8&siteId=25%2C12%2C14%2C13%2C15%2C11%2C10%2C65%2C231%2C2%2C4%2C1%2C20%2C22%2C23%2C21%2C163%2C28%2C253%2C62%2C63%2C68%2C71%2C80%2C51%2C58%2C117%2C93%2C250%2C142%2C148%2C139%2C146%2C147%2C192%2C193%2C211%2C216%2C219%2C197%2C198%2C217%2C212%2C3%2C89%2C165%2C40%2C91%2C43%2C114%2C111%2C252%2C224%2C7%2C90%2C59%2C57%2C84%2C52%2C81%2C140%2C194%2C18%2C19%2C16%2C143%2C39%2C42%2C6%2C29%2C35%2C37%2C70%2C79%2C54%2C61%2C60%2C67%2C87%2C95%2C88%2C119%2C251%2C196%2C195%2C240%2C241%2C242%2C86%2C55%2C64%2C53%2C66%2C82%2C48%2C221%2C222%2C223%2C30%2C73%2C33%2C41%2C113%2C237%2C238%2C239%2C243%2C5%2C74%2C78%2C56%2C206%2C31%2C235%2C17%2C34%2C36%2C38&mode=market&_=1524808327596'
+URL_SEARCH_GRADES = 'https://www.iauc.co.jp/ajax/search/grades?makerVehicleId%5B%5D=8-%EF%BD%B4%EF%BE%99%EF%BE%8C&siteId=25%2C12%2C14%2C13%2C15%2C11%2C10%2C65%2C231%2C2%2C4%2C1%2C20%2C22%2C23%2C21%2C163%2C28%2C253%2C62%2C63%2C68%2C71%2C80%2C51%2C58%2C117%2C93%2C250%2C142%2C148%2C139%2C146%2C147%2C192%2C193%2C211%2C216%2C219%2C197%2C198%2C217%2C212%2C3%2C89%2C165%2C40%2C91%2C43%2C114%2C111%2C252%2C224%2C7%2C90%2C59%2C57%2C84%2C52%2C81%2C140%2C194%2C18%2C19%2C16%2C143%2C39%2C42%2C6%2C29%2C35%2C37%2C70%2C79%2C54%2C61%2C60%2C67%2C87%2C95%2C88%2C119%2C251%2C196%2C195%2C240%2C241%2C242%2C86%2C55%2C64%2C53%2C66%2C82%2C48%2C221%2C222%2C223%2C30%2C73%2C33%2C41%2C113%2C237%2C238%2C239%2C243%2C5%2C74%2C78%2C56%2C206%2C31%2C235%2C17%2C34%2C36%2C38&ep=%2FmarketPrice%2FcarNames%2Ftypes%2Fgrades&_=1524808327598'
+URL_SEARCH_TYPES = 'https://www.iauc.co.jp/ajax/search/types?makerVehicleId%5B%5D=8-%EF%BD%B4%EF%BE%99%EF%BE%8C&siteId=25%2C12%2C14%2C13%2C15%2C11%2C10%2C65%2C231%2C2%2C4%2C1%2C20%2C22%2C23%2C21%2C163%2C28%2C253%2C62%2C63%2C68%2C71%2C80%2C51%2C58%2C117%2C93%2C250%2C142%2C148%2C139%2C146%2C147%2C192%2C193%2C211%2C216%2C219%2C197%2C198%2C217%2C212%2C3%2C89%2C165%2C40%2C91%2C43%2C114%2C111%2C252%2C224%2C7%2C90%2C59%2C57%2C84%2C52%2C81%2C140%2C194%2C18%2C19%2C16%2C143%2C39%2C42%2C6%2C29%2C35%2C37%2C70%2C79%2C54%2C61%2C60%2C67%2C87%2C95%2C88%2C119%2C251%2C196%2C195%2C240%2C241%2C242%2C86%2C55%2C64%2C53%2C66%2C82%2C48%2C221%2C222%2C223%2C30%2C73%2C33%2C41%2C113%2C237%2C238%2C239%2C243%2C5%2C74%2C78%2C56%2C206%2C31%2C235%2C17%2C34%2C36%2C38&ep=%2FmarketPrice%2FcarNames%2Ftypes&_=1524808327597'
+
+URL_CARLIST = 'https://www.iauc.co.jp/market/carlist'
+URL_CARLIST_NARROW = 'https://www.iauc.co.jp/ajax/list/carlist_narrow'
+URL_CARLIST_NEXTPAGE = 'https://www.iauc.co.jp/ajax/list/carlist?element=&sort_type=&limit=100&branch=&controller=marketvehicle&display_mode=&page_myListId=&page_requestId=&type=page&isHtml=true&preload=0&mode=table&referrer='
+
+COOKIE_CARLIST = 'iauc_lang=ja; _ga=GA1.3.1960749188.1524724940; _gid=GA1.3.162237504.1524724940; iauc_rf=0; iauc_limit_vehicle=15; iauc_limit_market_vehicle=100; iauc_search_period_vehicle=12; _gat=1;'
+QUERY_CARLIST = 'boxMakerScroll=0&domesticMakerScroll=106&foreignMakerScroll=200&boxTypeScroll=0&boxModelScroll=0&boxGradeScroll=0&attrColorScroll=0&makerAccordionButtonOpen=0&nameAccordionButtonOpen=0&maker%5B%5D=8&type%5B%5D=8-%7B%EF%BD%B4%EF%BE%99%EF%BE%8C%7D&searchPeriod=12&modelOfYearFrom=&modelOfYearTo=&mileageFrom=&mileageTo=&rateFrom=&rateTo=&vehicleInspection=&shiftType=&search_mode=maker'
+QUERY_CARLIST_2 = 'boxMakerScroll=0&domesticMakerScroll=0&foreignMakerScroll=0&boxTypeScroll=3735&boxModelScroll=0&boxGradeScroll=0&attrColorScroll=0&makerAccordionButtonOpen=0&nameAccordionButtonOpen=0&__tid=941d6bba79e0970136cdc8c45af4407818618&maker%5B%5D=0&type%5B%5D=0-%7B%EF%BE%80%EF%BE%9E%EF%BD%B2%EF%BE%85%7D&searchPeriod=12&modelOfYearFrom=&modelOfYearTo=&mileageFrom=&mileageTo=&rateFrom=&rateTo=&vehicleInspection=&shiftType=&search_mode=maker'

data/lib/weget/crawler.rb

@@ -0,0 +1,168 @@
+# coding: utf-8
+require 'rubygems'
+require 'nokogiri'
+require 'json'
+require './lib.rb'
+require './config.rb'
+
+ENV['SSL_CERT_FILE'] = File.join(File.dirname($0), 'cert.pem')
+exit if Object.const_defined?(:Ocra)
+#1 セットVist
+puts "1 セットVist  #{URL_SERVICE}"
+res = get(URL_SERVICE, nil)
+setCookie = res.header["set-cookie"].to_s
+cookies = setCookie.split(";")
+stmp = find_cookie(cookies, " path=/, _l_tm_stmp=")
+fuelrid = find_cookie(cookies, " path=/, fuelrid=")
+awselb = find_cookie(cookies, " path=/, AWSELB=")
+
+cookie = "iauc_lang=ja; _l_tm_stmp=#{stmp}; fuelrid=#{fuelrid}; AWSELB=#{awselb}"
+puts "1 セットVist cookie => #{cookie}"
+puts "======================================"
+
+#2　新規TID 
+# puts "2　新規TID  #{URL_TAB_CREATE}"
+# tab_id = ""
+# res = post(URL_TAB_CREATE, cookie, "old_tab_id=#{tab_id}&has_focus=true", nil)
+# fileName = "./out/tid.html"
+# File.open(fileName, 'w') { |file| file.write(res.body) } 
+# exit
+# xml = Nokogiri::XML(res.body)
+# tab_id = xml.xpath("//tab_id")[0].text
+xml = Nokogiri::HTML(res.body)
+tab_id = xml.xpath('//*[@id="slider"]/div[5]/a')[0]["href"]
+tab_id = tab_id.split('?__tid=')[1]
+puts "2　新規TID  tid => #{tab_id}"
+puts "======================================"
+
+#3　仮ログイン
+puts "3　仮ログイン  #{URL_NOTICE_PRE_LOGIN}"
+data = "id=#{USER_ID}&password=#{USER_PASS}&referer=normal"
+res = post(URL_NOTICE_PRE_LOGIN, cookie, data, nil)
+xml = Nokogiri::XML(res.body)
+item = xml.xpath("//item")[0].text
+puts "3　仮ログイン   item => #{item}"
+puts "======================================"
+
+#4　お知らせ
+puts "4　お知らせ1  #{URL_REQUIRED_NOTICE}"
+res = get("#{URL_REQUIRED_NOTICE}?id=#{item}&_=15247339838444", cookie)
+puts "======================================"
+
+#5　お知らせ
+puts "5　お知らせ2  #{URL_REQUIRED_NOTICE_TOP}"
+res = get("#{URL_REQUIRED_NOTICE_TOP}?id=#{item}&_=1524733983845", cookie)
+puts "======================================"
+
+#6　ログイン
+puts "6　ログイン  #{URL_LOGIN}"
+data = "id=#{USER_ID}&password=#{USER_PASS}&__tid=#{tab_id}"
+res = post(URL_LOGIN, cookie, data, "#{URL_LOGIN}?__tid=#{tab_id}")
+puts "======================================"
+
+begin
+    #相場検索⇒全選択⇒次へ=>イスズ⇒ｴﾙﾌ⇒次へ=>次ページ .... =>lastページ
+    now = Time.now.to_i.to_s
+    #7.1　ページVist 相場会場選択
+    puts "7.1　ページVist 相場会場選択  #{URL_MARKET}"
+    url = "#{URL_MARKET}?sess=clear&sess_m=clear?__tid=#{tab_id}"
+    res = post(url, cookie, "", "#{URL_LOGIN}?__tid=#{tab_id}")
+    fileName = "./out/#{now}.html"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.1　ページVist 相場会場選択 output => #{fileName}"
+    puts "======================================"
+
+    #7.2　相場会場検索
+    puts "7.2　相場会場検索  #{URL_SEARCH_MARKET}"
+    res = get("#{URL_SEARCH_MARKET}&__tid=#{tab_id}", cookie)
+    fileName = "./out/#{now}_maker.json"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.2　相場会場検索 output => #{fileName}"
+    puts "======================================"
+
+    #7.3　相場会場検索2
+    puts "7.3　相場会場検索2  #{URL_SEARCH_MARKET2}"
+    res = get("#{URL_SEARCH_MARKET2}&__tid=#{tab_id}", cookie)
+    fileName = "./out/#{now}_maker.html"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.3　相場会場検索2 output => #{fileName}"
+    puts "======================================"
+
+    #7.4　車両検索
+    puts "7.4　車両検索  #{URL_SEARCH_VEHICLE}"
+    res = get("#{URL_SEARCH_VEHICLE}&__tid=#{tab_id}", cookie)
+    fileName = "./out/#{now}_vehicle.json"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.4　車両検索 output => #{fileName}"
+    puts "======================================"
+
+    #7.5　グレード検索
+    puts "7.5　グレード検索  #{URL_SEARCH_GRADES}"
+    res = get("#{URL_SEARCH_GRADES}&__tid=#{tab_id}", cookie)
+    fileName = "./out/#{now}_grade.json"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.5　グレード検索 output => #{fileName}"
+    puts "======================================"
+
+    #7.6　タイプ検索
+    puts "7.6　タイプ検索  #{URL_SEARCH_TYPES}"
+    res = get("#{URL_SEARCH_TYPES}&__tid=#{tab_id}", cookie)
+    fileName = "./out/#{now}_type.json"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.6　タイプ検索 output => #{fileName}"
+    puts "======================================"
+
+    #7.8　ページcarlist　イスズ　ｴﾙﾌ page=1
+    puts "7.8　ページcarlist　イスズ　ｴﾙﾌ page=1  #{URL_CARLIST}"
+    cookie = "#{COOKIE_CARLIST} fuelrid=#{fuelrid}; AWSELB=#{awselb}; _l_tm_stmp=#{stmp}"
+    data = "__tid=#{tab_id}&#{QUERY_CARLIST}"
+    res = post(URL_CARLIST, cookie, data, "#{URL_SEARCH_MARKET2}&__tid=#{tab_id}")
+    fileName = "./out/#{now}_carlist.html"
+    File.open(fileName, 'w') { |file| file.write(res.body) }
+    puts "7.8　ページcarlist　イスズ　ｴﾙﾌ page=1 output => #{fileName}"
+    puts "======================================"
+
+    #7.9　ページcarlist　イスズ　ｴﾙﾌ page=2
+    page = 2
+    narrow_key = ""
+    data = "element=&narrow_data=&old_narrow_key=#{narrow_key}&__tid=#{tab_id}"
+    res = post(URL_CARLIST_NARROW, cookie, data, "#{URL_SEARCH_MARKET2}&__tid=#{tab_id}")
+    narrow_data = JSON.parse(res.body)
+    narrow_key = narrow_data["narrow_key"]
+    puts "7.9　ページcarlist　narrow_key =>  #{narrow_key}"
+    puts "7.9　ページcarlist　イスズ　ｴﾙﾌ page=#{page}  #{URL_CARLIST_NEXTPAGE}"
+    url = "#{URL_CARLIST_NEXTPAGE}&page=#{page}&narrow_key=#{narrow_key}&__tid=#{tab_id}&_=#{Time.now.to_i.to_s}"
+    res = get(url, cookie)
+    car_data = JSON.parse(res.body)
+    fileName = "./out/#{now}_carlist_page#{page}.html"
+    File.open(fileName, 'w') { |file| file.write(car_data["res_html"]) }
+    puts "7.9　ページcarlist　イスズ　ｴﾙﾌ count:#{car_data['count']} page=#{page} output => #{fileName}"
+    puts "======================================"
+
+    pageCnt = (car_data['count'].to_i / 100.00).ceil
+
+    (3..pageCnt).each do |p|
+       #7.9　ページcarlist　イスズ　ｴﾙﾌ page=3
+        page = p
+        data = "element=&narrow_data=&old_narrow_key=#{narrow_key}&__tid=#{tab_id}"
+        res = post(URL_CARLIST_NARROW, cookie, data, "#{URL_SEARCH_MARKET2}&__tid=#{tab_id}")
+        narrow_data = JSON.parse(res.body)
+        narrow_key = narrow_data["narrow_key"]
+        puts "7.10　ページcarlist　narrow_key =>  #{narrow_key}"
+        puts "7.10　ページcarlist　イスズ　ｴﾙﾌ page=#{page}  #{URL_CARLIST_NEXTPAGE}"
+        url = "#{URL_CARLIST_NEXTPAGE}&page=#{page}&narrow_key=#{narrow_key}&__tid=#{tab_id}&_=#{Time.now.to_i.to_s}"
+        res = get(url, cookie)
+        car_data = JSON.parse(res.body)
+        fileName = "./out/#{now}_carlist_page#{page}.html"
+        File.open(fileName, 'w') { |file| file.write(car_data["res_html"]) }
+        puts "7.10　ページcarlist　イスズ　ｴﾙﾌ count:#{car_data['count']} page=#{page} output => #{fileName}"
+        puts "======================================"
+    end
+rescue => error
+    puts error.message
+ensure
+    #10　ログアウト
+    puts "99　ログアウト  #{URL_LOGOUT}"
+    res = get("#{URL_LOGOUT}?__tid=#{tab_id}", cookie)
+    puts "======================================"
+end

data/lib/weget/lib.rb

@@ -0,0 +1,57 @@
+# coding: utf-8
+require 'net/http'
+require 'open-uri'
+require 'uri'
+
+USER_AGENTE = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko"
+
+def get(uri_str, cookie)
+    url = URI.parse(uri_str)
+    http = Net::HTTP.new(url.host, url.port)
+    http.use_ssl = true
+    #http.set_debug_output STDERR
+    req = Net::HTTP::Get.new(url.request_uri)
+    req["User-Agente"] = USER_AGENTE
+    if cookie != nil then
+        req["Cookie"] = cookie;
+    end
+    res = http.request(req)
+    case res
+    when Net::HTTPSuccess     then res
+    when Net::HTTPRedirection then res
+    else
+        res.error!
+    end
+end
+
+def post(url, cookie, data, referer)
+    uri = URI.parse(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    #http.set_debug_output STDERR
+    req = Net::HTTP::Post.new(uri.request_uri)
+    req["Content-Type"] = "application/x-www-form-urlencoded"
+    req["User-Agente"] = USER_AGENTE
+    req["Cookie"] = cookie
+    if referer != nil then
+        req["Referer"] = referer
+    end
+    req.body = data
+    res = http.request(req)
+    case res
+    when Net::HTTPSuccess     then res
+    when Net::HTTPRedirection then res
+    else
+      res.error!
+    end
+end
+
+def find_cookie(cookies, key)
+    fuelrid = ""
+    cookies.each do |cookie|
+        if(cookie.start_with?(key)) then
+            fuelrid = cookie.gsub(key, "")
+        end
+    end
+    fuelrid
+end

data/lib/weget/version.rb

@@ -1,3 +1,3 @@
 module Weget
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: weget
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - shouaya
@@ -45,13 +45,21 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".DS_Store"
 - ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- lib/.DS_Store
 - lib/weget.rb
+- lib/weget/.DS_Store
+- lib/weget/analysis.rb
+- lib/weget/cert.pem
+- lib/weget/config.rb
+- lib/weget/crawler.rb
+- lib/weget/lib.rb
 - lib/weget/version.rb
 - weget.gemspec
 homepage: https://shouaya.github.io