crowd_funding_parser 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 431eb1ced64d76041c587114664d712ad213869a
+  data.tar.gz: 2d0b7cb076f446a83b41171fb9f0af98e7d1a4ad
+SHA512:
+  metadata.gz: 27b2259f0552a50fda78ed86047a9ef3b5f7f2d28c9be8f10feed1d1106cdfe756d25cfc0a14989c88f80dfe3ef037b675ec86208e98a0b7265df98a5ba92f52
+  data.tar.gz: 62eede16e512955c3b3e0c8616402d8cbd8ff4938e6b5a6532b6a7064018b49e12c71ec1b561573b3611d4752822568f1fe8d58190468dafbd2f73fd45f4e394

data/.gitignore

@@ -0,0 +1,18 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+.DS_Store
+*/.DS_Store
+.DS_Store?
+*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crowd_funding_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Stan Low
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# CrowdFundingParser
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'crowd_funding_parser'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install crowd_funding_parser
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/crowd_funding_parser/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/crowd_funding_parser.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'crowd_funding_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "crowd_funding_parser"
+  spec.version       = CrowdFundingParser::VERSION
+  spec.authors       = ["BackerFounder"]
+  spec.email         = ["hello@backer-founder.com"]
+  spec.summary       = %q{A crowd-funding platform parser}
+  # spec.description   = %q{TODO: Write a longer description. Optional.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "parallel", "~> 1.3"
+  spec.add_runtime_dependency "nokogiri", "~> 1.6"
+end

data/lib/crowd_funding_parser/general.rb

@@ -0,0 +1,94 @@
+module CrowdFundingParser
+  module Parser
+    class General
+      def parse_tracking_data(doc, rel_url)
+        project_url = @url + rel_url
+        project = Hash.new
+        project['money_goal']      = get_money_goal(doc).to_i
+        project['money_pledged']   = get_money_pledged(doc).to_i
+        project['backer_count']    = get_backer_count(doc).to_i
+        project['last_time']       = get_last_time(doc)
+        project['status']          = get_status(project['last_time'])
+        # project['backer_list']     = get_backer_list(project_url)
+        project['fb_count']        = get_fb_count(doc).to_i
+        project['following_count'] = get_following_count(doc).to_i
+        project
+      end
+
+      def parse_content_data(doc, rel_url)
+        project_url = @url + rel_url
+        project                  = Hash.new
+        project['platform_project_id'] = get_id(rel_url)
+        project['title']         = get_title(doc)
+        project['url']           = project_url
+        project['summary']       = get_summary(doc)
+        project['category']      = get_category(doc)
+        project['creator_name']  = get_creator_name(doc)
+        project['creator_id']    = get_creator_id(doc)
+        project['creator_link']  = get_creator_link(doc)
+        project
+      end
+
+      def get_project_links(required_status = "online")
+        links = []
+        
+        @targets.each do |target|
+          doc = Nokogiri::HTML(target)
+          online_projects = doc.css(@item_css_class)
+
+          Parallel.map(online_projects, in_processes: 2 , in_threads: 4) do |project|
+            link_nodes = project.css("a:nth-child(1)")
+            status = get_status(get_string(project.css(@status_css_class)))
+            link = link_nodes.first["href"]
+            if status == "finished" && required_status == "finished"
+              links << link
+            elsif status == "online" && required_status == "online"
+              links << link  
+            elsif status == "preparing" && required_status == "preparing"
+              links << link
+            end
+          end
+        end
+
+        links
+      end
+
+      def get_project_log(url)
+        url.gsub!("#{@url}", "")
+        parse_tracking_data(url)
+      end
+
+      def get_project_content(url)
+        url.gsub!("#{@url}", "")
+        parse_content_data(url)
+      end
+
+      def get_full_project(url)
+        get_project_content(url).merge(get_project_log(url))
+      end
+
+      def get_doc_through_url(rel_url)
+        project_url = @url + rel_url
+        project_html = open(project_url)
+        Nokogiri::HTML(project_html)
+      end
+
+      private
+
+      def get_string(elements)
+        elements.first.text.strip
+      end
+
+      def money_string(money)
+        money.sub('$', '').sub(',', '').sub('NT', "")
+      end
+
+      def convert_time(left_time)
+        days = ((left_time / (60 * 60 * 24))).to_i
+        hours = ((left_time / (60 * 60)) % 24).to_i
+        minutes = ((left_time / 60) % 60).to_i
+        "#{days}天#{hours}小時#{minutes}分鐘"
+      end
+    end
+  end
+end

data/lib/crowd_funding_parser/parser/flyingv.rb

@@ -0,0 +1,90 @@
+require 'open-uri'
+
+module CrowdFundingParser
+  module Parser
+    class Flyingv < General
+      def initialize(*cat)
+        categories = cat.empty? ? ["designgoods", "media", "stageplay", "entertainment", "publish", "society", "technology", "food", "travel"] : cat
+        @url = "https://www.flyingv.cc"
+        @targets = []
+
+        categories.each do |category|
+          category_url = @url + "/category/#{category}"
+          @targets << open(category_url)
+        end
+
+        @item_css_class = ".portfolio-item"
+        @status_css_class = ".unit-time"
+      end
+
+      # for project's info
+      def get_id(rel_url)
+        rel_url.split("/").last
+      end
+
+      def get_title(doc)
+        get_string(doc.css(".page-title-wrapper").css(".pagesTitle"))
+      end
+
+      def get_category(doc)
+        doc.css(".page-title-wrapper").css(".pageDes").first.css("a").first.text
+      end
+
+      def get_creator_name(doc)
+        doc.css(".page-title-wrapper").css(".pageDes")[1].css("a").first.text.strip
+      end
+
+      def get_creator_id(doc)
+        doc.css(".page-title-wrapper").css(".pageDes")[1].css("a").first["href"].split("/").last
+      end
+
+      def get_creator_link(doc)
+        @url + doc.css(".profilemeta .imp a").first["href"]
+      end
+
+      def get_summary(doc)
+        doc.css(".project_content").first.text.to_s[0..500].strip
+      end
+
+      # for tracking
+
+      def get_money_goal(doc)
+        money_string(get_string(doc.css(".countdes .dt .white")))
+      end
+
+      def get_money_pledged(doc)
+        money_string(get_string(doc.css(".countdes .ut .rtt h3")))
+      end
+
+      def get_backer_count(doc)
+        get_string(doc.css(".countdes .dt .pull-right")).sub("人贊助", "")
+      end
+
+      def get_last_time(doc)
+        get_string(doc.css(".countdes .dt div:nth-child(2)")).sub("剩餘", "")
+      end
+
+      def get_status(last_time)
+        if last_time.match("已結束")
+          "finished"
+        elsif last_time.match("開始")
+          "preparing"
+        else
+          "online"
+        end
+      end
+
+      def get_fb_count(doc)
+        get_string(doc.css("#fbBtn .sharenumber"))
+      end
+
+      def get_following_count(doc)
+        get_string(doc.css(".sidebarprj h5")).sub("人追踨", "").sub("追蹤", "").strip
+      end
+
+      def get_backer_list(project_url)
+        []
+      end
+    end
+  end
+end

data/lib/crowd_funding_parser/parser/webackers.rb

@@ -0,0 +1,93 @@
+require 'open-uri'
+
+module CrowdFundingParser
+  module Parser
+    class Webackers < General
+      def initialize(*cat)
+        categories = cat.empty? ? ["ART", "PUBLICATION", "MUSIC", "DESIGN", "TECHNOLOGY", "ACG", "SURPRISE", "CHARITY", "VIDEO"] : cat
+        @url = "http://www.webackers.com"
+        @targets = []
+
+        categories.each do |category|
+          category_url = @url + "/Proposal/Browse?queryType=ALL&fundedStatus=ALL&category=#{category}"
+          @targets << open(category_url)
+        end
+
+        @item_css_class = ".cbp-item"
+        @status_css_class = "li.timeitem"
+      end
+
+      # for project's info
+      def get_id(rel_url)
+        rel_url.split("/").last
+      end
+
+      def get_title(doc)
+        get_string(doc.css(".case_header .case_title"))
+      end
+
+      def get_category(doc)
+       get_string(doc.css(".case_header .case_category a"))
+      end
+
+      def get_creator_name(doc)
+        get_string(doc.css(".headphoto_detail .name a"))
+      end
+
+      def get_creator_id(doc)
+        doc.css(".headphoto_detail .name a").first["href"].split("/").last
+      end
+
+      def get_creator_link(doc)
+        @url + doc.css(".headphoto_detail .name a").first["href"]
+      end
+
+      def get_summary(doc)
+        doc.css(".tab-content .description").first.text.to_s[0..500].strip
+      end
+
+      # for tracking
+
+      def get_money_goal(doc)
+        money_string(get_string(doc.css(".money_target")))
+      end
+
+      def get_money_pledged(doc)
+        money_string(get_string(doc.css(".money_now")))
+      end
+
+      def get_backer_count(doc)
+        get_string(doc.css(".tabbable li:nth-child(3) .badge.bg_no"))
+      end
+
+      def get_last_time(doc)
+        time_string = get_string(doc.css("article:nth-child(4) .panel-body span:nth-child(2)"))
+        last_seconds = (Time.parse(time_string) - Time.now)
+        last_seconds <= 0 ? "已結束" : convert_time(last_seconds)
+      end
+
+      def get_status(last_time)
+        if last_time.match("已結束") || last_time.match("已完成")
+          "finished"
+        elsif last_time.match("開始")
+          "preparing"
+        else
+          "online"
+        end
+      end
+
+      def get_fb_count(doc)
+        # get 0 because of ajax delay
+        get_string(doc.css(".fbBtn span.fb_share_count"))
+      end
+
+      def get_following_count(doc)
+        get_string(doc.css(".badge_share"))
+      end
+
+      def get_backer_list(project_url)
+        []
+      end
+    end
+  end
+end

data/lib/crowd_funding_parser/parser/zeczec.rb

@@ -0,0 +1,90 @@
+require 'open-uri'
+
+module CrowdFundingParser
+  module Parser
+    class Zeczec < General
+      def initialize
+        @url = "https://www.zeczec.com"
+        @targets = [open(@url + "/categories")]
+        @item_css_class = ".project-list .span4"
+        @status_css_class = ".meta span:nth-child(2)"
+      end
+
+      def get_id(rel_url)
+        rel_url.split("/").last
+      end
+
+      def get_title(doc)
+        get_string(doc.css("a.project-title"))
+      end
+
+      def get_category(doc)
+        doc.css(".project-meta").css("a")[0].text
+      end
+
+      def get_creator_name(doc)
+        doc.css(".project-meta").css("a")[1].text.strip
+      end
+
+      def get_creator_id(doc)
+        doc.css(".project-meta").css("a")[1]["href"].split("/").last
+      end
+
+      def get_creator_link(doc)
+        @url + doc.css(".creator .fly-center a").first["href"]
+      end
+
+      def get_summary(doc)
+        doc.css(".project-content").first.text.to_s[0..500].strip
+      end
+
+      # for tracking
+
+      def get_money_goal(doc)
+        if doc.css(".sidebar .project-notice strong").empty?
+          get_money_pledged(doc).to_i / get_percentage(doc).to_i * 100
+        else
+          money_string(get_string(doc.css(".sidebar .project-notice strong:nth-child(2)")))
+        end
+      end
+
+      def get_money_pledged(doc)
+        money_string(get_string(doc.css(".sidebar h3.num")))
+      end
+
+      def get_backer_count(doc)
+        get_string(doc.css("span.counter"))
+      end
+
+      def get_last_time(doc)
+        money_string(get_string(doc.css(".sidebar .row-fluid .span6:nth-child(2) h3.num")))
+      end
+
+      def get_percentage(doc)
+        money_string(get_string(doc.css(".sidebar .row-fluid .span6:nth-child(1) h3.num")))
+      end
+
+      def get_status(last_time)
+        if last_time.match("前") || last_time.match("達成")
+          "finished"
+        elsif last_time.match("開始")
+          "preparing"
+        else
+          "online"
+        end
+      end
+
+      def get_fb_count(doc)
+        ""
+      end
+
+      def get_following_count(doc)
+        ""
+      end
+
+      def get_backer_list(project_url)
+        []
+      end
+    end
+  end
+end

data/lib/crowd_funding_parser/version.rb

@@ -0,0 +1,3 @@
+module CrowdFundingParser
+  VERSION = "0.1.1"
+end

data/lib/crowd_funding_parser.rb

@@ -0,0 +1,6 @@
+require 'open-uri'
+require "crowd_funding_parser/version"
+require "crowd_funding_parser/general"
+require "crowd_funding_parser/parser/flyingv"
+require "crowd_funding_parser/parser/webackers"
+require "crowd_funding_parser/parser/zeczec"

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: crowd_funding_parser
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- BackerFounder
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: 
+email:
+- hello@backer-founder.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- crowd_funding_parser.gemspec
+- lib/crowd_funding_parser.rb
+- lib/crowd_funding_parser/general.rb
+- lib/crowd_funding_parser/parser/flyingv.rb
+- lib/crowd_funding_parser/parser/webackers.rb
+- lib/crowd_funding_parser/parser/zeczec.rb
+- lib/crowd_funding_parser/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A crowd-funding platform parser
+test_files: []