fly_parser 0.0.23 → 0.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35c2945cf908943eea6512e24e7b39106cf9ee9b
-  data.tar.gz: c8e48aa58849e9cc2696602f22a2b4a058fab8b3
+  metadata.gz: d7390fe6d8dd04f001c9d1df1d4876e828585438
+  data.tar.gz: 4baee214d519608b5ebf3cf891b16aea7acd9e2d
 SHA512:
-  metadata.gz: 1bc47c495a712b29615ca23935f61011db8607bc48a2ebe4131da250f4fcada371dec4540a4dc5e43f93c394ca5b043787fcfa4bcbf336cf4c77f942cba18a79
-  data.tar.gz: a62844515966e3af62cdad8ec7c50a57b68393def5f2f5049861cbc1bfd92b23f373712be63a8b786ef13b88d32b12260ca9242fad92518d4ab044d375284ba4
+  metadata.gz: 58133b0caf6bf9ec5b2714a4d29797c2156972f3701785dedf80a4eb9e484633db59a1807b07a9212b06ce687034d738b8b2cc86069ee3c1d3c432372cc82b7e
+  data.tar.gz: 216f6d754dd7ed4d8697885b2b1bf60b474d90763c5d7767daae999ce254384da889440d370c294947e572633d0accf9c697d74d0688eb606a80cf6995f1cbe0

data/lib/fly_parser.rb

@@ -116,8 +116,3 @@ module Parser
   end
 
 end
-
-# astrology
-# astro = Parser::Astrology.new('http://moj-znak-zodiaka.ru/')
-# astro.parse_in("Гороскоп на сентябрь 2014","big")
-# astro.parse_all

data/lib/fly_parser/base.rb

@@ -42,5 +42,13 @@ module Parser
         title: source['copyright_title']
       }
     end
+
+    def click(link)
+      agent = Mechanize.new
+      agent.ignore_bad_chunking = true
+      agent.pluggable_parser.default = Mechanize::Page
+      agent.click(link)
+    end
+
   end
-end
+end

data/lib/fly_parser/enable_source.rb

@@ -27,6 +27,10 @@ module Enable
     lambda {|item| item["parser"] = Parser::NewsAZ.new(item["url"], source: source)}
   end
 
+  def news_kz(source)
+    lambda {|item| item["parser"] = Parser::NewsKZ.new(item["url"], source: source)}
+  end
+
   def method_missing(meth, *args)
     prefix = "enable_"
     meth = meth.to_s
@@ -42,4 +46,4 @@ module Enable
   def iterate_sources(source, block)
     source["items"].each(&block)
   end
-end
+end

data/lib/fly_parser/sources/news-kz.rb

@@ -0,0 +1,42 @@
+module Parser
+  class NewsKZ < Base
+    def initialize(source, options = {})
+      @delay = 2
+      super
+    end
+
+    def parse_all
+      links = @source.search("li.c__news_item a:first")
+      
+      links.map do |link|
+        page = click(link)
+        begin
+        title = page.search(".c__article_caption").text()
+
+        content_wrapper = page.search('.c__article_text')
+        
+        image_wrapper = content_wrapper.search('.wp-caption img').first
+        next unless image_wrapper
+        poster_image = image_wrapper.attributes['src'].value
+        
+        content_wrapper.search('.wp-caption').remove()
+        content_wrapper.search('.c__article_mistake').remove()
+        content_wrapper.search('p[style="display:none"]').remove()
+        content_wrapper.search("a").remove()
+        content_wrapper.search("span:contains(Копирование)").remove()
+        
+        full_desc = content_wrapper.to_html
+        full_desc.gsub!(/<iframe.*><\/iframe>/, '')
+        copyright = "<p>Source: <a href='#{@copyright[:url]}'>#{@copyright[:title]}</a></p>"
+        content = full_desc + copyright
+        
+        {title: title, content: content, poster_image: poster_image}
+
+        rescue Exception => e
+          puts e.message
+          next
+        end 
+      end.compact
+    end
+  end
+end

data/lib/fly_parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = "0.0.23"
+  VERSION = "0.0.24"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fly_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.23
+  version: 0.0.24
 platform: ruby
 authors:
 - Ruslan Korolev
@@ -135,11 +135,11 @@ files:
 - lib/fly_parser/enable_source.rb
 - lib/fly_parser/logo.txt
 - lib/fly_parser/mechanize_fix.rb
-- lib/fly_parser/sources/astrology.rb
 - lib/fly_parser/sources/exercise.rb
 - lib/fly_parser/sources/fitness.rb
 - lib/fly_parser/sources/news-az.rb
 - lib/fly_parser/sources/news-fr.rb
+- lib/fly_parser/sources/news-kz.rb
 - lib/fly_parser/sources/news-nl.rb
 - lib/fly_parser/sources/news.rb
 - lib/fly_parser/sources/sport.rb
@@ -170,3 +170,4 @@ signing_key:
 specification_version: 4
 summary: Fly parser
 test_files: []
+has_rdoc: 

data/lib/fly_parser/sources/astrology.rb

@@ -1,67 +0,0 @@
-require 'hashie'
-module Parser
-  class Astrology
-    
-    def initialize(source)
-      @zodiacs = ['Овен','Телец','Близнецы','Рак','Лев','Дева','Весы','Скорпион','Стрелец','Козерог','Водолей','Рыбы']
-      @source = Parser.connect(source)
-
-      small_titles = ["Гороскоп на сегодня", "Гороскоп на завтра", "Гороскоп на неделю"]
-      big_titles = ["Гороскопы на месяц", "Гороскоп на 2014 год", "Гороскоп на 2014 год зеленой Лошади", "Гороскоп на сентябрь 2014"]
-      @titles = Hashie::Mash.new 
-      @titles.small = small_titles
-      @titles.big = big_titles
-    end
-
-    def parse_in(text = "Гороскоп на сегодня", date = 'small')
-      @text = text
-      @date = date
-      parse_content
-    end  
-
-    def parse_content
-      zodiac_links.map do |item|
-        link = item.link
-        zodiac = item.zodiac
-        @page = Parser.http(link.value)
-        content = (@date == 'small' ? parse_small : parse_big)
-        result = Hashie::Mash.new 
-        result.zodiac = zodiac
-        result.content = content
-        result
-      end
-    end
-
-    def parse_all
-      small_content = @titles.small.map { |title| {title: title, content: parse_in(title,"small")} }
-      big_content = @titles.big.map { |title| {title: title, content: parse_in(title,"big")} }
-      
-      small_content.concat big_content
-    end  
-
-    private
-
-    def current_page
-      @source.link_with(:text => @text).click
-    end
-
-    def zodiac_links
-      @zodiacs.map do |z|
-        result = Hashie::Mash.new
-        result.link = current_page.search("a:contains('#{z}')")[0].attributes["href"]
-        result.zodiac = z
-        result
-      end
-    end  
-    
-    def parse_small
-      @page.css('#main').children().reject do |el| 
-        (el.attributes['class'].value == 'lp50' || el.attributes['class'].value == 'rp50' || el.attributes['class'].value == "space" if el.attributes['class'] != nil) || ['img','br','b','h1'].include?(el.name) || ["\n","\n\n"].include?(el.text)
-      end.join
-    end  
-    
-    def parse_big
-      @page.css('#main .lp50').text()
-    end
-  end  
-end