google_news 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d82fabbe3456707329c9d55f22433ff8cf03ecc2528f35f805f3a33c5a006e4d
-  data.tar.gz: 05ac38bfcec8edc401ba73d13f4b95bc45d5caebee93e72c6626d90a38d2324c
+  metadata.gz: 6168625d6e729f8fa19728bb27caed64749d600c55550edb80c4370775b2eb96
+  data.tar.gz: 91c7cdb1d9b6ad61e621f3c914cbf117c2dd7aabc86bc5e62e4856a500d2f6f0
 SHA512:
-  metadata.gz: 81a86f5527947e62673d615d6efb9fe7c2bc3c4206cfa60a805f88e8eb6cc3b360a9d73b30b232c87a551d2527130a2316694f07bc597fa8f7c21c72ad60f402
-  data.tar.gz: 4640d37f2b90cee7a4081ca023ecb892da77bdabbff2b3d19049dbf674bb6651d0b916db4b08227d06cb1ca8aee7129002b7cc8b546e88811a8134ee19239c09
+  metadata.gz: 447e5eb18577d9cb5c0cfce07e68c1527224de5113838ea173d4a2b2f1fe84ff1a84bd58000358b0a5d8d8a257bdb43b0ae5fd13f7be4ad00b5c505021d3764f
+  data.tar.gz: 2736ba54da2962201979a53d57029c15d52cb4c2985ea0690e6731887cfff04aab582848e7d860a87c2f410a71ab6126c5d84a40942f4082ac7c104b7c28d91e

data/.ruby-version

@@ -0,0 +1 @@
+3.4.6

data/CHANGELOG.md

@@ -1,2 +1,5 @@
+# 0.1.0
+- refactored headline output to have parsed description(s) as Array with Hashes instead of one string
+
 # 0.0.1
 - initial alpha release with basic features, most is untested

data/Gemfile

@@ -2,5 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in google_news.gemspec
 gemspec
-
-

data/README.md

@@ -73,7 +73,25 @@ require 'google_news'
 
 results = GoogleNews.headlines n: 5 # get top 5 news in default language (country: us, language: en)
 puts results
-# => [{"title"=>"Title of the news article", "link"=>"https://link.to/the/article", "pubDate"=>"Wed, 01 Jan 2024 00:00:00 GMT", "source"=>"Source Name"}, ...]
+# => [
+# {
+#   :title =>    "Title of the news article", 
+#   :link =>     "https://link.to/the/article", 
+#   :pub_date => Time("Wed, 01 Jan 2024 00:00:00 GMT"), 
+#   :descriptions => [
+#     {
+#       :title => "First article, like :title",
+#       :link => "https://link.to/the/first/article",
+#       :author => "Author Name",
+#     },
+#    {
+#      :title => "Second article, same topic, similiar to first",
+#      :link => "https://link.to/the/second/article",
+#     :author => "Another Author Name",
+#    },
+#    ...
+#  ],
+# ]
 ```
 
 

data/Rakefile

@@ -14,4 +14,20 @@ task :console do
   Pry.start
 end
 
+namespace :example do
+  task :headlines do
+    require_relative 'lib/google_news'
+    headlines = GoogleNews.headlines(n: 10, language: 'de', country: 'de')
+    headlines.each do |item|
+      puts "Title: #{item[:title]}"
+      puts "Link: #{item[:link]}"
+      puts "Published At: #{item[:pub_date].class}"
+      puts "Descriptions: #{item[:descriptions]}"
+      puts "-" * 40
+    end
+  end
+
+end
+
+
 RSpec::Core::RakeTask.new(:spec)

data/google_news.gemspec

@@ -4,15 +4,15 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'google_news/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "google_news"
-  spec.version       = GoogleNews::VERSION
-  spec.executables   = %w[google_news]
-  spec.authors       = ["Matthäus J. N. Beyrle"]
-  spec.email         = ["google_news.gemspec@mail.magynhard.de"]
+  spec.name = "google_news"
+  spec.version = GoogleNews::VERSION
+  spec.executables = %w[google_news]
+  spec.authors = ["Matthäus J. N. Beyrle"]
+  spec.email = ["google_news.gemspec@mail.magynhard.de"]
 
-  spec.summary       = %q{Get Google News headlines from Ruby or the command line.}
-  spec.homepage      = "https://github.com/magynhard/google_news"
-  spec.license       = "MIT"
+  spec.summary = %q{Get Google News headlines from Ruby or the command line.}
+  spec.homepage = "https://github.com/magynhard/google_news"
+  spec.license = "MIT"
 
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
   spec.metadata['allowed_push_host'] = "https://rubygems.org"
@@ -26,13 +26,14 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'rss',      '>= 0.2.6'
+  spec.add_runtime_dependency 'rss', '>= 0.2.6'
   spec.add_runtime_dependency 'open-uri', '>= 0.1.0'
 
-  spec.add_development_dependency 'bundler',  '>= 2.7.1'
-  spec.add_development_dependency 'rake',     '~> 12.0'
-  spec.add_development_dependency 'rspec',    '~> 3.0'
-  spec.add_development_dependency 'pry',     '~> 0.15.2'
-  spec.add_development_dependency 'fiddle',   '~> 1.1.8'
-  spec.add_development_dependency 'ostruct',  '~> 0.6.3'
+  spec.add_development_dependency 'bundler', '>= 2.7.1'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry', '~> 0.15.2'
+  spec.add_development_dependency 'fiddle', '~> 1.1.8'
+  spec.add_development_dependency 'ostruct', '~> 0.6.3'
+  spec.add_development_dependency 'nokogiri', '~> 1.18'
 end

data/lib/google_news/version.rb

@@ -1,3 +1,3 @@
 module GoogleNews
-  VERSION = '0.0.2'.freeze
+  VERSION = '0.1.0'.freeze
 end

data/lib/google_news.rb

@@ -5,14 +5,15 @@
 require 'open-uri'
 require 'rss'
 require 'uri'
+require 'nokogiri'
 
 require_relative 'google_news/version'
 
 module GoogleNews
   HEADLINES_RSS = 'https://news.google.com/news/rss'.freeze
-  TOPICS_RSS    = 'https://news.google.com/news/rss/headlines/section/topic/'.freeze
-  GEO_RSS       = 'https://news.google.com/news/rss/headlines/section/geo/'.freeze
-  SEARCH_RSS    = 'https://news.google.com/rss/search?q='.freeze
+  TOPICS_RSS = 'https://news.google.com/news/rss/headlines/section/topic/'.freeze
+  GEO_RSS = 'https://news.google.com/news/rss/headlines/section/geo/'.freeze
+  SEARCH_RSS = 'https://news.google.com/rss/search?q='.freeze
 
   TOPICS = %w[WORLD NATION BUSINESS TECHNOLOGY ENTERTAINMENT SPORTS SCIENCE HEALTH].freeze
 
@@ -201,17 +202,18 @@ module GoogleNews
   # Convert an RSS item to a hash
   #
   # @param [RSS::Rss::Channel::Item] item
-  # @return [Hash] with keys :title, :link, :pub_date, :description, and :raw_item
+  # @return [Hash] with keys :title, :link, :pub_date, :descriptions [:title, :link, :author], and :raw_item
   #
   def self.item_to_hash(item)
     {
       title: item.title,
       link: extract_link(item),
       pub_date: (item.respond_to?(:pubDate) ? item.pubDate : nil),
-      description: (item.respond_to?(:description) ? item.description : nil),
+      descriptions: (item.respond_to?(:description) ? parse_description(item.description) : nil),
       raw_item: item
     }
   end
+
   #
   # Extract the link from an RSS item
   #
@@ -225,6 +227,42 @@ module GoogleNews
     nil
   end
 
+  #
+  # Description can consist of a <a> tag with a link, containing the title, followed by a <font> tag with the authors name.
+  #
+  # But it can also consist of a bunch of news, inside a <ol> list, containing <li> items with a <a> tag, followed by a <font> tag each.
+  #
+  # This method creates a array of hashes with :title, :link and :author keys for each news item found in the description.
+  #
+  # @param [String] description as HTML
+  # @return [Array<Hash>] array of news items with :title, :link and :author keys
+  #
+  def self.parse_description(description)
+    return nil if description.nil?
+    doc = Nokogiri::HTML(description)
+    news_items = []
+    if doc.at_css('ol')
+      doc.css('ol li').each do |li|
+        a_tag = li.at_css('a')
+        font_tag = li.at_css('font')
+        news_items << {
+          title: a_tag ? a_tag.text : nil,
+          link: a_tag ? a_tag['href'] : nil,
+          author: font_tag ? font_tag.text : nil
+        }
+      end
+    else
+      a_tag = doc.at_css('a')
+      font_tag = doc.at_css('font')
+      news_items << {
+        title: a_tag ? a_tag.text : nil,
+        link: a_tag ? a_tag['href'] : nil,
+        author: font_tag ? font_tag.text : nil
+      }
+    end
+    news_items
+  end
+
   #
   # Default User-Agent string for HTTP requests
   #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google_news
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Matthäus J. N. Beyrle
@@ -121,6 +121,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.6.3
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
 email:
 - google_news.gemspec@mail.magynhard.de
 executables:
@@ -131,6 +145,7 @@ files:
 - ".gitattributes"
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -169,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.1
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Get Google News headlines from Ruby or the command line.
 test_files: []