crowd_funding_parser 0.1.1 → 0.1.2

data/lib/crowd_funding_parser/parser/flyingv.rb

@@ -1,90 +1,90 @@
-require 'open-uri'
-
 module CrowdFundingParser
   module Parser
     class Flyingv < General
-      def initialize(*cat)
-        categories = cat.empty? ? ["designgoods", "media", "stageplay", "entertainment", "publish", "society", "technology", "food", "travel"] : cat
-        @url = "https://www.flyingv.cc"
-        @targets = []
+      MethodBuilder.set_methods do
+        insert_parser "Flyingv"
 
-        categories.each do |category|
-          category_url = @url + "/category/#{category}"
-          @targets << open(category_url)
+        set_variable do
+          @platform_url = "https://www.flyingv.cc"
+          @time_regex = /(\d{4}\/\d{2}\/\d{2}).+(\d{4}\/\d{2}\/\d{2})/
         end
 
-        @item_css_class = ".portfolio-item"
-        @status_css_class = ".unit-time"
-      end
+        set_method :get_title do |doc|
+          get_string(doc.css(".page-title-wrapper").css(".pagesTitle"))
+        end
 
-      # for project's info
-      def get_id(rel_url)
-        rel_url.split("/").last
-      end
+        set_method :get_category do |doc|
+          doc.css(".page-title-wrapper").css(".pageDes").first.css("a").first.text
+        end
 
-      def get_title(doc)
-        get_string(doc.css(".page-title-wrapper").css(".pagesTitle"))
-      end
+        set_method :get_creator_name do |doc|
+          doc.css(".page-title-wrapper").css(".pageDes")[1].css("a").first.text.strip
+        end
 
-      def get_category(doc)
-        doc.css(".page-title-wrapper").css(".pageDes").first.css("a").first.text
-      end
+        set_method :get_creator_id do |doc|
+          doc.css(".page-title-wrapper").css(".pageDes")[1].css("a").first["href"].split("/").last
+        end
 
-      def get_creator_name(doc)
-        doc.css(".page-title-wrapper").css(".pageDes")[1].css("a").first.text.strip
-      end
+        set_method :get_creator_link do |doc|
+          @platform_url + doc.css(".profilemeta .imp a").first["href"]
+        end
 
-      def get_creator_id(doc)
-        doc.css(".page-title-wrapper").css(".pageDes")[1].css("a").first["href"].split("/").last
-      end
+        set_method :get_summary do |doc|
+          doc.css(".project_content").first.text.to_s[0..500].strip
+        end
 
-      def get_creator_link(doc)
-        @url + doc.css(".profilemeta .imp a").first["href"]
-      end
+        set_method :get_start_date do |doc|
+          text = get_string(doc.css(".col-xs-4.sidebarprj")).gsub(/\n/, "")
+          @time_regex.match(text)[1]
+        end
 
-      def get_summary(doc)
-        doc.css(".project_content").first.text.to_s[0..500].strip
-      end
+        set_method :get_end_date do |doc|
+          text = get_string(doc.css(".col-xs-4.sidebarprj")).gsub(/\n/, "")
+          @time_regex.match(text)[2]
+        end
 
-      # for tracking
+        set_method :get_region do |doc|
+          "Taiwan"
+        end
 
-      def get_money_goal(doc)
-        money_string(get_string(doc.css(".countdes .dt .white")))
-      end
+        set_method :get_money_goal do |doc|
+          money_string(get_string(doc.css(".countdes .dt .white")))
+        end
 
-      def get_money_pledged(doc)
-        money_string(get_string(doc.css(".countdes .ut .rtt h3")))
-      end
+        set_method :get_money_pledged do |doc|
+          money_string(get_string(doc.css(".countdes .ut .rtt h3")))
+        end
 
-      def get_backer_count(doc)
-        get_string(doc.css(".countdes .dt .pull-right")).sub("人贊助", "")
-      end
+        set_method :get_backer_count do |doc|
+          get_string(doc.css(".countdes .dt .pull-right")).sub("人贊助", "")
+        end
 
-      def get_last_time(doc)
-        get_string(doc.css(".countdes .dt div:nth-child(2)")).sub("剩餘", "")
-      end
+        set_method :get_left_time do |doc|
+          get_string(doc.css(".countdes .dt div:nth-child(2)")).sub("剩餘", "")
+        end
 
-      def get_status(last_time)
-        if last_time.match("已結束")
-          "finished"
-        elsif last_time.match("開始")
-          "preparing"
-        else
-          "online"
+        set_method :get_status do |left_time|
+          if left_time.match("已結束")
+            "finished"
+          elsif left_time.match("開始")
+            "preparing"
+          else
+            "online"
+          end
         end
-      end
 
-      def get_fb_count(doc)
-        get_string(doc.css("#fbBtn .sharenumber"))
-      end
+        set_method :get_fb_count do |doc|
+          get_string(doc.css("#fbBtn .sharenumber"))
+        end
 
-      def get_following_count(doc)
-        get_string(doc.css(".sidebarprj h5")).sub("人追踨", "").sub("追蹤", "").strip
-      end
+        set_method :get_following_count do |doc|
+          get_string(doc.css(".sidebarprj h5")).sub("人追踨", "").sub("追蹤", "").strip
+        end
 
-      def get_backer_list(project_url)
-        []
+        set_method :get_currency_string do |result|
+          "twd"
+        end
       end
     end
   end
-end
+end

data/lib/crowd_funding_parser/parser/hereo.rb

@@ -0,0 +1,99 @@
+module CrowdFundingParser
+  module Parser
+    class Hereo < General
+      def initialize
+        @platform_url = "http://www.hereo.cc/"
+        @item_css_class = ".project-list ul li"
+        @status_css_class = ".projectImg .info .inner .detail span:nth-child(1)"
+      end
+
+      def get_lists
+        [HTTParty.get(@platform_url + "/project-list.php")]
+      end
+
+      def get_id(project_url)
+        project_url.split("pid=").last
+      end
+
+      MethodBuilder.set_methods do
+        insert_parser "Hereo"
+
+        set_variable do
+          @platform_url = "http://www.hereo.cc/"
+        end
+
+        set_method :get_title do |doc|
+          get_string(doc.css(".container .text h3"))
+        end
+
+        set_method :get_category do |doc|
+          get_string(doc.css(".contentMain .projectTag"))
+        end
+
+        set_method :get_creator_name do |doc|
+          get_string(doc.css(".user-info .user .name h4 a"))
+        end
+
+        set_method :get_creator_id do |doc|
+          doc.css(".user-info .user .name h4 a")[0]["href"].match(/mid=(\d+)/)[1]
+        end
+
+        set_method :get_creator_link do |doc|
+          @platform_url + doc.css(".user-info .user .name h4 a")[0]["href"]
+        end
+
+        set_method :get_summary do |doc|
+          doc.css(".container div.text").first.text.gsub(/\s/, "")
+        end
+
+        set_method :get_start_date do |doc|
+        end
+
+        set_method :get_end_date do |doc|
+          doc.css(".projectInfo .detail .inner p").text.match(/\d{4}\/\d{2}\/\d{2}/).to_s
+        end
+
+        set_method :get_region do |doc|
+          "Taiwan"
+        end
+
+        set_method :get_money_pledged do |doc|
+          money_string(doc.css(".projectInfo .funded .inner .number strong").text.match(/[0-9,]+/).to_s)
+        end
+
+        set_method :get_money_goal do |doc|
+          money_string(get_string(doc.css(".sidebar h3.num")))
+        end
+
+        set_method :get_backer_count do |doc|
+          doc.css(".projectInfo .table .numberOfPeople .inner strong").text
+        end
+
+        set_method :get_left_time do |doc|
+          raw_string = doc.css(".projectInfo .table .time .inner").text.gsub(/\s/, "")
+          match_data = raw_string.match(/(\d+).*(天|小時)/)
+          match_data[1] + match_data[2]
+        end
+
+        set_method :get_status do |left_time|
+          if left_time.match("集資中")
+            "online"
+          elsif left_time.match("結束") || left_time.match("成功") || left_time.match(/\d+/).to_s == "0"
+            "finished"
+          else
+            "online"
+          end
+        end
+
+        set_method :get_following_count do
+          doc.css("strong#track-count").text
+        end
+
+        set_method :get_currency_string do |result|
+          "twd"
+        end
+      end
+    end
+  end
+end
+

data/lib/crowd_funding_parser/parser/kickstarter.rb

@@ -0,0 +1,158 @@
+require "json"
+require "iconv"
+
+module CrowdFundingParser
+  module Parser
+    class Kickstarter < General
+      def initialize
+        @platform_url = "https://www.kickstarter.com"
+        @category_ids = [1, 3, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 26]
+        @parse_method = :doc
+      end
+
+      def get_all_categories(status = "online")
+        status_code = get_status_code(status)
+        jsons = get_category_project_jsons(status_code)
+        jsons.flatten.compact!
+        categories = []
+        Parallel.each(jsons, in_precesses: 2, in_threads: 5) do |json|
+          category = { id: json["category"]["id"], name: json["category"]["name"], parent_id: json["category"]["parent_id"]}
+          categories << category
+        end
+        categories.uniq
+      end
+
+      def get_project_links(status = "online")
+        status_code = get_status_code(status)
+
+        jsons = @category_ids.map do |category_id|
+          category_jsons = get_category_project_jsons(status_code, category_id)
+        end.flatten.compact
+
+        Parallel.map(jsons, in_precesses: 2, in_threads: 5) do |json|
+          unless json["state"] != "live" && json["pledged"].to_i == 0
+            if json["state"] == status_code
+              project_url = json["urls"]["web"]["project"]
+            end
+          end
+        end
+      end
+
+      def get_category_project_jsons(status_code = "live", category_id = 0)
+        jsons = []
+
+        Parallel.each(1..200, in_precesses: 2, in_threads: 5) do |i|
+          begin
+            api_url = get_projects_page_api(i, status_code, category_id)
+            json = get_json_through_url(api_url)["projects"]
+            jsons << json
+          rescue Exception => e
+            Parallel::Stop
+          end
+        end
+        jsons
+      end
+
+      private
+
+      def get_project_page_api(project_url)
+        project_url.split("?").first + ".json"
+      end
+
+      def get_projects_page_api(page = 1, status_code = "live", category_id = 0)
+        "https://www.kickstarter.com/projects/search.json?page=#{page}&state=#{status_code}&category_id=#{category_id}"
+      end
+
+      def get_project_search_doc_api(name)
+        "https://www.kickstarter.com/projects/search.json?term=#{name}"
+      end
+
+      def get_status_code(status)
+        case status
+        when "online"
+          "live"
+        when "finished"
+          "successful"
+        else
+          "live"
+        end
+      end
+
+      MethodBuilder.set_methods do
+        insert_parser "Kickstarter"
+
+        set_variable do
+          @platform_url = "https://www.kickstarter.com"
+        end
+
+        set_method :get_title do |doc|
+          get_string(doc.css(".NS_projects__header h2 .green-dark"))
+        end
+
+        set_method :get_category do |doc|
+          get_string(doc.css(".container-flex .h5 a.grey-dark:nth-child(2) b"))
+        end
+
+        set_method :get_creator_name do |doc|
+          get_string(doc.css(".NS_projects__creator .col-8>h5 a.remote_modal_dialog"))
+        end
+
+        set_method :get_creator_id do |doc|
+          creator_path = doc.css(".NS_projects__creator .col-8>h5 a.remote_modal_dialog").first["href"]
+          creator_path.split("/")[-3]
+        end
+
+        set_method :get_creator_link do |doc|
+          @platform_url + doc.css(".NS_projects__creator .col-8>h5 a.remote_modal_dialog").first["href"]
+        end
+
+        set_method :get_summary do |doc|
+          get_string(doc.css(".container-flex .col-8 .mobile-hide p.h3.mb3"))
+        end
+
+        set_method :get_end_date do |doc|
+          doc.css(".NS_projects__deadline_copy p.grey-dark time[datetime]").try(:first).try(:[], "datetime")
+        end
+
+        set_method :get_region do |doc|
+          get_string(doc.css(".container-flex .h5 a.grey-dark:nth-child(1) b"))
+        end
+
+        set_method :get_money_pledged do |doc|
+          doc.css("div[data-pledged]").first["data-pledged"]
+        end
+
+        set_method :get_money_goal do |doc|
+          doc.css("div[data-pledged]").first["data-goal"]
+        end
+
+        set_method :get_backer_count do |doc|
+          doc.css("div[data-backers-count]").first["data-backers-count"]
+        end
+
+        set_method :get_left_time do |doc|
+          end_date = doc.css("div[data-end_time]").try(:first).try(:[], "data-end_time") || Time.now.to_s
+          last_seconds = Time.parse(end_date) - Time.now
+          last_day = last_seconds.to_i / 86400
+          if last_day <= 0
+            "已結束"
+          else
+            last_day.to_s + "天"
+          end
+        end
+
+        set_method :get_status do |left_time|
+          if left_time == "已結束"
+            "finished"
+          else
+            "online"
+          end
+        end
+
+        set_method :get_currency_string do |doc|
+          doc.css("data[data-currency]")[0]["data-currency"]
+        end
+      end
+    end
+  end
+end

data/lib/crowd_funding_parser/parser/taobao.rb

@@ -0,0 +1,150 @@
+require "json"
+require "iconv"
+
+module CrowdFundingParser
+  module Parser
+    class Taobao < General
+      def initialize
+        @parse_method = :json
+        @url = "http://hi.taobao.com/market/hi/detail2014.php?id="
+      end
+
+      def get_project_links(status = "online")
+        status_code = get_status_code(status)
+        jsons = get_total_jsons(status_code)
+        links = []
+
+        Parallel.each(jsons, in_precesses: 2, in_threads: 5) do |json|
+          project_id = json["id"].to_s
+          project_url = @url + project_id
+          links << project_url
+        end
+        links
+      end
+
+      def get_status_code(status)
+        case status
+        when "online"
+          1
+        when "preparing"
+          3
+        when "finished"
+          2
+        else
+          1
+        end
+      end
+
+      def get_total_jsons(status = 1)
+        urls = get_total_json_apis(status)
+        jsons = []
+        urls.each do |url|
+          page_json = get_json_through_url(url)
+          json = page_json["data"]
+          jsons += json
+        end
+        jsons
+      end
+
+      def get_total_json_apis(status = 1)
+        page_count = get_total_page(status)
+        total_urls = []
+        page_count.to_i.times do |i|
+          total_urls << get_projects_page_api(i + 1, status)
+        end
+        total_urls
+      end
+
+      def get_total_page(status = 1)
+        url = "http://hstar-hi.alicdn.com/dream/ajax/getProjectList.htm?page=1&pageSize=20&projectType=&type=6&status=#{status}"
+        json = get_json_through_url(url)
+        page_count = json["pageTotal"]
+      end
+
+      private
+
+      def get_project_api(project_id)
+        "http://hstar-hi.alicdn.com/dream/ajax/getProjectForDetail.htm?id=#{project_id}"
+      end
+
+      def get_projects_page_api(page = 1, status = 1)
+        "http://hstar-hi.alicdn.com/dream/ajax/getProjectList.htm?page=#{page}&pageSize=20&projectType=&type=6&status=#{status}"
+      end
+
+      def get_id(project_url)
+        project_url.split("id=").last
+      end
+
+      MethodBuilder.set_methods do
+        insert_parser "Taobao"
+
+        set_method :get_title do |result|
+          result["data"]["name"]
+        end
+
+        set_method :get_creator_name do |result|
+          raw_creator_name = result["data"]["person"]["name"]
+          @parser.encode_gbk_to_utf(raw_creator_name)
+        end
+
+        set_method :get_summary do |result|
+          raw_summary = result["data"]["desc"]
+          @parser.encode_gbk_to_utf(raw_summary)
+        end
+
+        set_method :get_start_date do |result|
+          result["data"]["begin_date"]
+        end
+
+        set_method :get_end_date do |result|
+          result["data"]["end_date"]
+        end
+
+        set_method :get_region do |result|
+          "China"
+        end
+
+        set_method :get_money_goal do |result|
+          result["data"]["target_money"]
+        end
+
+        set_method :get_money_pledged do |result|
+          result["data"]["curr_money"]
+        end
+
+        set_method :get_backer_count do |result|
+          result["data"]["support_person"]
+        end
+
+        set_method :get_left_time do |result|
+          result["data"]["remain_day"]
+        end
+
+        set_method :get_status do |left_time|
+          if left_time == "0"
+            "finished"
+          else
+            "online"
+          end
+        end
+
+        set_method :get_following_count do |result|
+          result["data"]["focus_count"]
+        end
+
+        set_method :get_currency_string do |result|
+          "cny"
+        end
+
+        set_method :encode_gbk_to_utf do |string|
+          begin
+            Iconv.conv("utf-8//ignore", "gb2312//ignore", string)
+          rescue Exception => e
+            puts e
+            string
+          end
+        end
+      end
+    end
+  end
+end