fly_parser 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a4b60890098babea5cdf0a19b2ab7f72207bce93
-  data.tar.gz: a5cf7fdba982c30f622f2c048ea5ef08d6a4565a
+  metadata.gz: 04dc5a5ee6aefcaad08a66b4d86d418d2e2d6621
+  data.tar.gz: 8145b7beb20e9b7047c2422e3172a14b25218267
 SHA512:
-  metadata.gz: 4d9bfcfbc33a83acbf053cca55206f27377a1f6ab62ead285def08fc43ef659ea61aee533c5c743b21ee06b8d7c22e79b77942370ce76ace69270e14f359f5fd
-  data.tar.gz: f4b199bf649c121cbf1883e00acf451742dd25830076bc3b55a62258e149d1c314026e447580a3c8e2eab3d42b6648e5b9adeb763056aa3bec10322c62d6c288
+  metadata.gz: 4283c77559952bdcc2ed404e9b629c0263b806d9d0a776fed4d198674b6e92fb5601f65942fb9b29297fc7e2a42599ceba6740f2fcedcbce2bda0ec1432f9e0b
+  data.tar.gz: 37b9f632c4a943defa36e6d5d6a666ed3d3735379d8e0221b10afb881e64d4df096b92f9fac75d215a7cfe7cf833b502713feba5c298212385fe6daf2a21eed4

data/lib/fly_parser/enable_source.rb

@@ -0,0 +1,41 @@
+# add new parser source here
+module Enable
+  def fitness(source)
+    lambda do |item|
+      item["parser"] = Parser::Exercise.new(item["url"], source: source) and next if item["type"] == "exercises"
+      item["parser"] = Parser::Fitness.new(item["url"], source: source)
+    end
+  end
+
+  def news(source)
+    lambda { |item| item["parser"] = Parser::News.new(item["url"], source: source) }
+  end
+
+  def local(source)
+    lambda { |item| item["parser"] = Parser::News.new(item["file"], {type: :file, source: source}) }
+  end
+
+  def news_nl(source)
+    lambda { |item| item["parser"] = Parser::NewsNl.new(item["url"], source: source) }
+  end
+
+  def news_fr(source)
+    lambda {|item| item["parser"] = Parser::NewsFr.new(item["url"], source: source)}
+  end
+
+  def method_missing(meth, *args)
+    prefix = "enable_"
+    meth = meth.to_s
+    if meth.start_with?(prefix)
+      meth_name = meth.split(prefix).last
+      proc = send(meth_name, *args)
+      iterate_sources(*args, proc)
+    else
+      raise "Unknown method #{meth} in Enable class, ssory !"
+    end
+  end
+
+  def iterate_sources(source, block)
+    source["items"].each(&block)
+  end
+end

data/lib/fly_parser/sources/news-fr.rb

@@ -0,0 +1,39 @@
+module Parser
+  class NewsFr < XmlBase
+    def initialize(source, options = {})
+      super
+    end
+
+    def parse_all
+      items = @source.search('//item')
+      # # last_date = Time.now - 2.years # for dev 2 years
+      # # select! or reject! is not exists for Nokogiri#NodeSet
+      # # items = items.select {|item| item.xpath('pubDate').first.content() > last_date }
+      items.map do |item|
+        title = item.xpath('title/text()').text()
+        date = item.xpath('pubdate').first.content()
+
+        link = item.xpath('link/following-sibling::text()[1]').first
+        page = Nokogiri::HTML(open(link))
+
+        next if page.search('figure.img img').first.nil?
+
+        poster_image = page.search('.article-long figure.img img').first.attributes['src'].value
+        full_desc = page.search('.article-long .bd')
+        full_desc.search('.modification').remove()
+        full_desc.search('script').remove()
+        full_desc.search('.ft').remove()
+        full_desc.search('a').remove_attr('href')
+        full_desc.search('.twitter-tweet').remove()
+
+        desc = full_desc.inner_html
+        desc.gsub! /h2|h1|h3/, 'h4'
+        # remove href attributes
+        #full_desc = full_desc.text().gsub(/<a href="([a-zA-Z:\/\.\d\-]*)">(.*)<\/a>/,'<a>\2</a>')
+        copyright = "<p>Source: <a href='#{@copyright[:url]}'>#{@copyright[:title]}</a></p>"
+        content = desc + copyright
+        {title: title, content: content, poster_image: poster_image}
+      end.compact
+    end
+  end
+end

data/lib/fly_parser.rb

@@ -34,6 +34,7 @@ end
 
 module Parser
   class << self
+    include Enable if defined? Enable
     # Get HTTP Source
     def http(url)
       Nokogiri::HTML(open(url))
@@ -41,6 +42,7 @@ module Parser
 
     def connect(url)
       agent = Mechanize.new
+      agent.pluggable_parser.default = Mechanize::Page
       agent.get(url)
     end
 
@@ -93,29 +95,10 @@ module Parser
       File.read(LOGO_PATH)
     end
 
-    # choose parser for source here
     def init_parser(source)
-      case source["source"]
-      when "fitness"
-        source["items"].each do |item|
-          item["parser"] = Parser::Exercise.new(item["url"], source: source) and next if item["type"] == "exercises"
-          item["parser"] = Parser::Fitness.new(item["url"], source: source)
-        end
-      when "news"
-        source["items"].each do |item|
-          item["parser"] = Parser::News.new(item["url"], source: source)
-        end
-      when "local"
-        if source["enabled"]
-          source["items"].each do |item|
-            item["parser"] = Parser::News.new(item["file"], {type: :file, source: source})
-          end
-        end
-      when "news-nl"
-        source["items"].each do |item|
-          item["parser"] = Parser::NewsNl.new(item["url"], source: source)
-        end
-      end
+      source_type = source["source"].gsub('-', '_')
+      prefix = "enable_"
+      send(prefix + source_type, source)
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fly_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Ruslan Korolev
@@ -118,11 +118,13 @@ files:
 - lib/fly_parser.rb
 - lib/fly_parser/base.rb
 - lib/fly_parser/config_example.yml
+- lib/fly_parser/enable_source.rb
 - lib/fly_parser/logo.txt
 - lib/fly_parser/mechanize_fix.rb
 - lib/fly_parser/sources/astrology.rb
 - lib/fly_parser/sources/exercise.rb
 - lib/fly_parser/sources/fitness.rb
+- lib/fly_parser/sources/news-fr.rb
 - lib/fly_parser/sources/news-nl.rb
 - lib/fly_parser/sources/news.rb
 - lib/fly_parser/sources/sport.rb