rfsp 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7de216178f224cec07a036bc0cd0acfc73c137cd
+  data.tar.gz: a1bf51d69f55ed977601bf2796f54ed9971cb42b
+SHA512:
+  metadata.gz: b20aa78fc5213162cdfc3042e5dc51f3626cbc2067eed32547ebd8cf41d7a0a2eb152b1f62ceca3d87a737e5a983443e3776196990a29d584d232b3dc3db210f
+  data.tar.gz: 33ba15273b3b6a6850e4d960fb928b4deabe8538c3d57d037fc47a2de22340187eec755a93a4905f388558f32f6d6bf1cd4561050e6d04c2593ac02e612732aa

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Leonid
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# RFSP
+
+Loads RSS from www.fl.ru, www.weblancer.net, freelansim.ru and normalize project info.
+
+## Installation
+
+    $ gem install rfsp
+
+## Usage
+
+```ruby
+
+projects = RFSP::Fl.parse_rss
+projects = RFSP::Weblancer.parse_rss
+projects = RFSP::Freelansim.parse_rss
+```
+
+Project contains info:
+
+1. `uri`
+1. `id`
+1. `published`
+1. `title`
+1. `body`
+1. `category` - ex. `Дизайн / Логотипы, Дизайн / Фирменный стиль`, not availiable for freelansim.ru
+1. `budget`
+    1. `origin` - originally parsed string
+    2. `amount` - integer
+    3. `currency` - ex. 'rur, 'usd'
+1. `tags` - only for freelansim.ru
+
+For freelansim to get budget and tags you should call `RFSP::Freelansim.parse_page project` - it'll get info from project page.
+
+Each class have methods:
+
+* `::feed` - returns feed parsed by Feedjira
+* `::update` - reloads cached feed

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task default: :spec
+task test: :spec
+

data/lib/rfsp.rb

@@ -0,0 +1,10 @@
+require 'rfsp/version'
+require 'rfsp/fl'
+require 'rfsp/client'
+require 'rfsp/freelansim'
+require 'rfsp/weblancer'
+
+module RFSP
+
+end
+

data/lib/rfsp/client.rb

@@ -0,0 +1,18 @@
+require 'nokogiri'
+require 'rest_client'
+require 'open-uri'
+require 'feedjira'
+
+class RFSP::Client
+  UA = 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0'
+
+  def self.get uri
+    # response = RestClient.get(uri, user_agent: UA)
+    # raise "Response code is #{response.code}" if response.code != 200
+    Nokogiri::HTML open(uri, "User-Agent" => UA)
+  end
+
+  def self.feed uri
+    Feedjira::Feed.fetch_and_parse uri, user_agent: UA
+  end
+end

data/lib/rfsp/fl.rb

@@ -0,0 +1,46 @@
+require_relative 'client'
+require 'ostruct'
+
+class RFSP::Fl
+  RSS_URI = 'https://www.fl.ru/rss/all.xml'
+
+  class << self
+
+    def parse_rss
+      projects = []
+      feed.entries.each do |e|
+        p = OpenStruct.new
+        p.uri = e.url
+        p.id = e.url.match(/(\d+)/)[1].to_i
+        p.published = e.published
+        p.body = e.summary
+        p.title = e.title
+        p.site = 'fl'
+        p.category = e.categories.map{|c| c.gsub(/<\/?.+?>/, '')}.join ', '
+        budget_regexp = / \(Бюджет: (\d+) +(руб|\$)\.?\)\Z/
+        if m = p.title.match(budget_regexp)
+          budget = OpenStruct.new
+          budget.origin = m[0]
+          budget.origin[' ('] = ''
+          budget.origin[')'] = ''
+          budget.amount = m[1].to_i
+          budget.currency = m[2] == '$' ? 'usd' : m[2] == 'руб' ? 'rur' : nil
+          p.budget = budget
+          p.title.sub! budget_regexp, ''
+        end
+        projects << p
+      end
+      projects
+    end
+
+    def feed
+      @@feed ||= update
+    end
+
+    def update
+      @@feed = RFSP::Client.feed RSS_URI
+    end
+
+  end
+
+end

data/lib/rfsp/freelansim.rb

@@ -0,0 +1,51 @@
+require_relative 'client'
+require 'ostruct'
+
+class RFSP::Freelansim
+  RSS_URI = 'http://freelansim.ru/rss/tasks'
+
+  class << self
+
+    def parse_rss
+      projects = []
+      feed.entries.each do |e|
+        p = OpenStruct.new
+        p.uri = e.url
+        p.id = e.url.match(/(\d+)/)[1].to_i
+        p.published = e.published
+        p.body = e.summary
+        p.title = e.title
+        p.site = 'freelansim'
+        projects << p
+      end
+      projects
+    end
+
+
+    def parse_budget_and_tags project
+      doc = RFSP::Client.get project.uri
+      price = doc.search('.task__price .count').first
+      if price
+        budget = OpenStruct.new
+        budget.origin = price.text
+        budget.amount = budget.origin.match(/([\d ]+)/)[1].gsub(' ', '').to_i
+        budget.currency = 'rur' if budget.origin['руб.']
+      end
+      project.budget = budget
+      project.tags = doc.search('.tags a').map(&:text).join ', '
+      project
+    end
+
+    alias :parse_page :parse_budget_and_tags
+
+    def feed
+      @@feed ||= update
+    end
+
+    def update
+      @@feed = RFSP::Client.feed RSS_URI
+    end
+
+  end
+
+end

data/lib/rfsp/version.rb

@@ -0,0 +1,3 @@
+module RFSP
+  VERSION = "0.9.0"
+end

data/lib/rfsp/weblancer.rb

@@ -0,0 +1,46 @@
+require_relative 'client'
+require 'ostruct'
+require 'nokogiri'
+
+class RFSP::Weblancer
+  RSS_URI = 'http://www.weblancer.net/rss/projects.rss'
+
+  class << self
+
+    def parse_rss
+      projects = []
+      feed.entries.each do |e|
+        p = OpenStruct.new
+        p.site = 'weblancer'
+        p.uri = e.url
+        p.id = e.url.match(/(\d+)/)[1].to_i
+        p.published = e.published
+        p.body = e.summary
+        p.title = e.title
+        body_doc = Nokogiri::HTML p.body
+        p.category = body_doc.search('a').select{|c| c.attr('href')['projects/?category_id=']}.map(&:text).join ', '
+        if amount2 = body_doc.search('.amount_2').first
+          budget = OpenStruct.new
+          budget.origin = amount2.text
+          m = budget.origin.match(/(\d+) (\w+)/)
+          budget.amount = m[1].to_i
+          budget.currency = m[2].downcase
+          p.budget = budget
+        end
+        p.body.sub!(/\n.+<br \/><br \/>/, '')
+        projects << p
+      end
+      projects
+    end
+
+    def feed
+      @@feed ||= update
+    end
+
+    def update
+      @@feed = RFSP::Client.feed RSS_URI
+    end
+
+  end
+
+end

data/rfsp.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rfsp/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'rfsp'
+  spec.version       = RFSP::VERSION
+  spec.authors       = ['Ibrahim Ivanko']
+  spec.email         = ['ibrahim-ivanko@mdtlpr.com']
+  spec.summary       = %q{Russian Freelance Site Parser}
+  spec.description   = %q{Get projects from www.fl.ru www.weblancer.net freelansim.ru}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'rest_client', '~> 1.0'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.0'
+  spec.add_runtime_dependency 'feedjira', '~> 1.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rspec-core', '~> 3.0'
+  spec.add_development_dependency 'rspec-mocks'
+  spec.add_development_dependency 'rspec-expectations', '~> 3.0'
+  spec.add_development_dependency 'rr', '~> 1.0'
+end

data/spec/rfsp/fl_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe RFSP::Fl do
+
+  describe 'projects' do
+    let(:projects){RFSP::Fl.parse_rss}
+    it 'loads 30 projects from rss' do
+      expect(projects.count).to eq 60
+    end
+
+    it 'initilize id and uri of projects' do
+      expect(projects[0].id.class).to eq Fixnum
+      expect(projects[0].uri.class).to eq String
+      expect(projects[0].uri).to_not be_empty
+      expect(projects[0].id).to be > 0
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rfsp'
+require 'pry'
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: rfsp
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Ibrahim Ivanko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: feedjira
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-expectations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Get projects from www.fl.ru www.weblancer.net freelansim.ru
+email:
+- ibrahim-ivanko@mdtlpr.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rfsp.rb
+- lib/rfsp/client.rb
+- lib/rfsp/fl.rb
+- lib/rfsp/freelansim.rb
+- lib/rfsp/version.rb
+- lib/rfsp/weblancer.rb
+- rfsp.gemspec
+- spec/rfsp/fl_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Russian Freelance Site Parser
+test_files:
+- spec/rfsp/fl_spec.rb
+- spec/spec_helper.rb