crowd_funding_parser 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 431eb1ced64d76041c587114664d712ad213869a
-  data.tar.gz: 2d0b7cb076f446a83b41171fb9f0af98e7d1a4ad
+  metadata.gz: a49010d36a36cab50f0c001bb1d368934c1126a3
+  data.tar.gz: 177a84fa3773c046fd6e4488c3470aae83c25cef
 SHA512:
-  metadata.gz: 27b2259f0552a50fda78ed86047a9ef3b5f7f2d28c9be8f10feed1d1106cdfe756d25cfc0a14989c88f80dfe3ef037b675ec86208e98a0b7265df98a5ba92f52
-  data.tar.gz: 62eede16e512955c3b3e0c8616402d8cbd8ff4938e6b5a6532b6a7064018b49e12c71ec1b561573b3611d4752822568f1fe8d58190468dafbd2f73fd45f4e394
+  metadata.gz: 5e3129df45328fb30f828ce77de424813926dae5d169da8ca66e0e3c55dcf5f5590e02bebf2810be137c78d2e2fbe59b251250b453e39c064678a9422045392a
+  data.tar.gz: 028b83e89377e5c631893be152b902e4f6430c187d43d5fedc5e4e18b67d0dfdfc80417c0d66e39d25a9026f51ef5432bc0cdd3d188fe6e376b680e3d4fb94aa

data/.gitignore

@@ -15,4 +15,4 @@ mkmf.log
 .DS_Store
 */.DS_Store
 .DS_Store?
-*.gem
+*.gem

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+cache: bundler
+
+rvm:
+  - 2.1.2
+
+script: 'bundle exec rspec'
+
+notifications:
+  slack:
+    rooms:
+      - backer-founder:gS9GQrOqRKRogUEU5ajHYFZ9#crowd-trail
+  email:
+    recipients:
+      - stan001212@gmail.com
+    on_failure: change
+    on_success: never

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in crowd_funding_parser.gemspec
 gemspec
+
+gem "pry-rails"

data/Guardfile

@@ -0,0 +1,11 @@
+guard 'rspec', cmd: "bundle exec rspec" do
+  # watch /lib/ files
+  watch(%r{^lib/(.+).rb$}) do |m|
+    "spec/#{m[1]}_spec.rb"
+  end
+
+  # watch /spec/ files
+  watch(%r{^spec/(.+).rb$}) do |m|
+    "spec/#{m[1]}.rb"
+  end
+end

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/BackerFounder/crowd_funding_parser.svg)](https://travis-ci.org/BackerFounder/crowd_funding_parser)
+
 # CrowdFundingParser
 
 TODO: Write a gem description

data/Rakefile

@@ -1,2 +1,10 @@
+require "rspec/core/rake_task"
 require "bundler/gem_tasks"
+# Default directory to look in is `/specs`
+# Run with `rake spec`
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = ['--color', '--format', 'nested']
+end
+
+task default: :spec
 

data/crowd_funding_parser.gemspec

@@ -18,8 +18,20 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "iconv"
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "vcr", "~> 2.9.3"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-nc"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-remote"
+  spec.add_development_dependency "pry-nav"
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_runtime_dependency "parallel", "~> 1.3"
   spec.add_runtime_dependency "nokogiri", "~> 1.6"
+  spec.add_runtime_dependency "httparty", "~> 0.13.3"
 end

data/lib/crowd_funding_parser.rb

@@ -4,3 +4,7 @@ require "crowd_funding_parser/general"
 require "crowd_funding_parser/parser/flyingv"
 require "crowd_funding_parser/parser/webackers"
 require "crowd_funding_parser/parser/zeczec"
+require "crowd_funding_parser/parser/taobao"
+require "crowd_funding_parser/parser/kickstarter"
+require "crowd_funding_parser/parser/hereo"
+require "crowd_funding_parser/parser/an9"

data/lib/crowd_funding_parser/general.rb

@@ -1,50 +1,53 @@
+require 'httparty'
+require "active_support/all"
+require_relative "method_builder"
+
 module CrowdFundingParser
   module Parser
     class General
-      def parse_tracking_data(doc, rel_url)
-        project_url = @url + rel_url
+      include HTTParty
+
+      def parse_tracking_data(result, project_url)
         project = Hash.new
-        project['money_goal']      = get_money_goal(doc).to_i
-        project['money_pledged']   = get_money_pledged(doc).to_i
-        project['backer_count']    = get_backer_count(doc).to_i
-        project['last_time']       = get_last_time(doc)
-        project['status']          = get_status(project['last_time'])
-        # project['backer_list']     = get_backer_list(project_url)
-        project['fb_count']        = get_fb_count(doc).to_i
-        project['following_count'] = get_following_count(doc).to_i
+        project["money_goal"]      = get_money_goal(result).to_i
+        project["money_pledged"]   = get_money_pledged(result).to_i
+        project["backer_count"]    = get_backer_count(result).to_i
+        project["left_time"]       = get_left_time(result)
+        project["status"]          = get_status(project["left_time"])
+        project["fb_count"]        = get_fb_count(result).to_i
+        project["following_count"] = get_following_count(result).to_i
         project
       end
 
-      def parse_content_data(doc, rel_url)
-        project_url = @url + rel_url
-        project                  = Hash.new
-        project['platform_project_id'] = get_id(rel_url)
-        project['title']         = get_title(doc)
-        project['url']           = project_url
-        project['summary']       = get_summary(doc)
-        project['category']      = get_category(doc)
-        project['creator_name']  = get_creator_name(doc)
-        project['creator_id']    = get_creator_id(doc)
-        project['creator_link']  = get_creator_link(doc)
+      def parse_content_data(result, project_url)
+        project                    = Hash.new
+        project["platform_project_id"] = get_id(project_url)
+        project["title"]           = get_title(result)
+        project["url"]             = project_url
+        project["summary"]         = get_summary(result)
+        project["category"]        = get_category(result)
+        project["creator_name"]    = get_creator_name(result)
+        project["creator_id"]      = get_creator_id(result)
+        project["creator_link"]    = get_creator_link(result)
+        project["currency_string"] = get_currency_string(result)
+        project["start_date"]      = get_start_date(result)
+        project["end_date"]        = get_end_date(result)
+        project["region"]          = get_region(result)
         project
       end
 
       def get_project_links(required_status = "online")
         links = []
-        
-        @targets.each do |target|
+
+        get_lists.each do |target|
           doc = Nokogiri::HTML(target)
           online_projects = doc.css(@item_css_class)
 
           Parallel.map(online_projects, in_processes: 2 , in_threads: 4) do |project|
             link_nodes = project.css("a:nth-child(1)")
             status = get_status(get_string(project.css(@status_css_class)))
-            link = link_nodes.first["href"]
-            if status == "finished" && required_status == "finished"
-              links << link
-            elsif status == "online" && required_status == "online"
-              links << link  
-            elsif status == "preparing" && required_status == "preparing"
+            link = @platform_url + link_nodes.first["href"]
+            if status == required_status
               links << link
             end
           end
@@ -53,42 +56,49 @@ module CrowdFundingParser
         links
       end
 
-      def get_project_log(url)
-        url.gsub!("#{@url}", "")
-        parse_tracking_data(url)
+      def get_result(project_url)
+        if @parse_method == :json
+          project_id = get_id(project_url)
+          project_api = get_project_api(project_id)
+          get_json_through_url(project_api)
+        else
+          get_doc_through_url(project_url)
+        end
+      end
+
+      def get_doc_through_url(project_url)
+        project_html = HTTParty.get(project_url)
+        Nokogiri::HTML(project_html)
       end
 
-      def get_project_content(url)
-        url.gsub!("#{@url}", "")
-        parse_content_data(url)
+      def get_json_through_url(project_url)
+        httparty_url = HTTParty.get(project_url)
+        json = JSON.load(httparty_url.body)
       end
 
-      def get_full_project(url)
-        get_project_content(url).merge(get_project_log(url))
+      def get_project(project_url)
+        result = get_result(project_url)
+        parse_content_data(result, project_url).merge parse_tracking_data(result, project_url)
       end
 
-      def get_doc_through_url(rel_url)
-        project_url = @url + rel_url
-        project_html = open(project_url)
-        Nokogiri::HTML(project_html)
+      def get_id(project_url)
+        rel_url = get_rel_url(project_url)
+        rel_url.split("/").last.split("?").first
       end
 
       private
 
-      def get_string(elements)
-        elements.first.text.strip
+      def get_rel_url(url)
+        url.gsub("#{@platform_url}", "")
       end
 
-      def money_string(money)
-        money.sub('$', '').sub(',', '').sub('NT', "")
-      end
-
-      def convert_time(left_time)
-        days = ((left_time / (60 * 60 * 24))).to_i
-        hours = ((left_time / (60 * 60)) % 24).to_i
-        minutes = ((left_time / 60) % 60).to_i
-        "#{days}天#{hours}小時#{minutes}分鐘"
+      def method_missing(m, *args, &block)
+        if m.to_s.match(/get/)
+          ""
+        else
+          super
+        end
       end
     end
   end
-end
+end

data/lib/crowd_funding_parser/method_builder.rb

@@ -0,0 +1,47 @@
+module MethodBuilder
+  def self.set_methods(&block)
+    ParserMethodProxy.new.instance_eval(&block)
+  end
+
+  class ParserMethodProxy
+    def insert_parser(inserted_class)
+      @parser_class = "CrowdFundingParser::Parser::#{inserted_class}".constantize
+      @parser = @parser_class.new
+    end
+
+    def set_variable(&block)
+      block.call
+    end
+
+    def set_method(method_name, &block)
+      @parser_class.send(:define_method, method_name) do |arg|
+        begin
+          block.call(arg)
+        rescue Exception => e
+          puts "Error #{e.message}"
+          puts e.backtrace.first
+          ""
+        end
+      end
+    end
+
+    def method_missing(m, *args, &block)
+      ""
+    end
+
+    def get_string(elements)
+      elements.first.text.strip
+    end
+
+    def money_string(money)
+      money.gsub("$","").gsub(",", "").gsub("NT", "")
+    end
+
+    def convert_time(left_time)
+      days = ((left_time / (60 * 60 * 24))).to_i
+      hours = ((left_time / (60 * 60)) % 24).to_i
+      minutes = ((left_time / 60) % 60).to_i
+      "#{days}天#{hours}小時#{minutes}分鐘"
+    end
+  end
+end

data/lib/crowd_funding_parser/parser/an9.rb

@@ -0,0 +1,196 @@
+require "json"
+require 'open-uri'
+require "iconv"
+
+module CrowdFundingParser
+  module Parser
+    class An9 < General
+      def initialize
+        @url = "http://www.an9.com.tw/Dream/"
+        @status_css_class = ".sideCon>a"
+      end
+
+      def get_main_categories(add_categories)
+        add_categories.select { |c| c[:parent_id].nil? }
+      end
+
+      def get_project_links(required_status = "online")
+        links = []
+        error_count = 0
+        not_found_count = 0
+        Parallel.each(1..100000, in_precesses: 2, in_threads: 5, progress: "Get #{self} links") do |i|
+          begin
+            link = @url + i.to_s
+            project = get_doc_through_url(link)
+            not_found_message = project.css(".actMsg p")
+            if not_found_message.present? && get_string(not_found_message).match(/不存在/)
+              not_found_count += 1
+            else
+              status = get_status(get_string(project.css(@status_css_class)))
+
+              if status == required_status
+                links << link
+              end
+              not_found_count = 0
+              error_count = 0
+            end
+          rescue Exception => e
+            error_count += 1
+            raise Parallel::Break if not_found_count >= 50 || error_count >= 50
+          end
+        end
+
+        links
+      end
+
+      private
+
+      def get_title(result)
+        if @parse_method == :doc
+          get_string(result.css(".NS_projects__header h2 .green-dark"))
+        else
+          result["name"]
+        end
+      end
+
+      def get_category(result)
+        if @parse_method == :doc
+          get_string(result.css(".container-flex .h5 a.grey-dark:nth-child(2) b"))
+        else
+          result["category"]["name"]
+        end
+      end
+
+      def get_creator_name(result)
+        if @parse_method == :doc
+          get_string(result.css(".NS_projects__creator .col-8>h5 a.remote_modal_dialog"))
+        else
+          result["creator"]["name"]
+        end
+      end
+
+      def get_creator_id(result)
+        if @parse_method == :doc
+          creator_link = result.css(".NS_projects__creator .col-8>h5 a.remote_modal_dialog").first["href"]
+          creator_link.split("/")[-3]
+        else
+          result["creator"]["id"]
+        end
+      end
+
+      def get_creator_link(result)
+        if @parse_method == :doc
+          creator_link = @url + result.css(".NS_projects__creator .col-8>h5 a.remote_modal_dialog").first["href"]
+        else
+          result["creator"]["urls"]["web"]["user"]
+        end
+      end
+
+      def get_summary(result)
+        if @parse_method == :doc
+          get_string(result.css(".container-flex .col-8 .mobile-hide p.h3.mb3"))
+        else
+          result["blurb"]
+        end
+      end
+
+      def get_start_date(result)
+        if @parse_method == :doc
+          # no start date on page
+        else
+          Time.at(result["launched_at"])
+        end
+      end
+
+      def get_end_date(result)
+        if @parse_method == :doc
+          result.css(".NS_projects__deadline_copy p.grey-dark time[datetime]")[0]["datetime"]
+        else
+          time = Time.at(result["deadline"])
+        end
+      end
+
+      def get_region(result)
+        if @parse_method == :doc
+          get_string(result.css(".container-flex .h5 a.grey-dark:nth-child(1) b"))
+        else
+          result["location"]["displayable_name"]
+        end
+      end
+
+      # for tracking
+
+      def get_money_goal(result)
+        if @parse_method == :doc
+          result.css("div[data-pledged]").first["data-goal"]
+        else
+          result["goal"]
+        end
+      end
+
+      def get_money_pledged(result)
+        if @parse_method == :doc
+          result.css("div[data-pledged]").first["data-pledged"]
+        else
+          result["pledged"]
+        end
+      end
+
+      def get_backer_count(result)
+        if @parse_method == :doc
+          result.css("div[data-backers-count]").first["data-backers-count"]
+        else
+          result["backers_count"]
+        end
+      end
+
+      def get_left_time(result)
+        if @parse_method == :doc
+          end_date = result.css("div[data-end_time]").first["data-end_time"]
+          last_seconds = Time.parse(end_date) - Time.now
+        else
+          last_seconds = result["deadline"].to_i - Time.now.to_i
+        end
+        last_day = last_seconds.to_i / 86400
+        if last_day <= 0
+          "已結束"
+        else
+          last_day.to_s + "天"
+        end
+      end
+
+      def get_status(button_text)
+        case button_text
+        when /贊助/
+          "online"
+        when /喜歡/
+          "voting"
+        when /結束|成功/
+          "finished"
+        else
+          "online"
+        end
+      end
+
+      def get_fb_count(result)
+        
+      end
+
+      def get_following_count(result)
+        
+      end
+
+      def get_backer_list(project_url)
+        []
+      end
+
+      def get_currency_string(result)
+        if @parse_method == :doc
+          result.css("data[data-currency]")[0]["data-currency"]
+        else
+          result["currency"]
+        end
+      end
+    end
+  end
+end