google_news 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df4beeb3e71a539fd40e36aff03536fd8239976510093a0af1f7c8cbc91df797
-  data.tar.gz: dbc67f83eed942876a9ac46e7763de6d19dbebdd135fe6d9ca09867ea0bc2282
+  metadata.gz: 6168625d6e729f8fa19728bb27caed64749d600c55550edb80c4370775b2eb96
+  data.tar.gz: 91c7cdb1d9b6ad61e621f3c914cbf117c2dd7aabc86bc5e62e4856a500d2f6f0
 SHA512:
-  metadata.gz: 68e12a403a5a857760ae821d4b8a6d9532b4825fbf7302a0cdfa40753945730ead7d0788bb459e5f67aa716c83be85d48bbf98e15fb53b434c1556e29f26676f
-  data.tar.gz: 695d594b22921aabc3ca570407c34bd9d47d227c188f39ad001b3c5d12f3ff1a6c515fdf9d87b5a3c3a1646ada27e5e76101159b654da89721d58a771e93c048
+  metadata.gz: 447e5eb18577d9cb5c0cfce07e68c1527224de5113838ea173d4a2b2f1fe84ff1a84bd58000358b0a5d8d8a257bdb43b0ae5fd13f7be4ad00b5c505021d3764f
+  data.tar.gz: 2736ba54da2962201979a53d57029c15d52cb4c2985ea0690e6731887cfff04aab582848e7d860a87c2f410a71ab6126c5d84a40942f4082ac7c104b7c28d91e

data/.gitignore

@@ -12,4 +12,4 @@
 *.gem
 
 # rspec failure tracking
-.rspec_status
+.rspec_status

data/.rspec

@@ -1,2 +1,3 @@
 --format documentation
 --color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+3.4.6

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 3.3.5
+before_install: gem install bundler -v 2.7.1

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# 0.1.0
+- refactored headline output to have parsed description(s) as Array with Hashes instead of one string
+
+# 0.0.1
+- initial alpha release with basic features, most is untested

data/README.md

@@ -73,7 +73,25 @@ require 'google_news'
 
 results = GoogleNews.headlines n: 5 # get top 5 news in default language (country: us, language: en)
 puts results
-# => [{"title"=>"Title of the news article", "link"=>"https://link.to/the/article", "pubDate"=>"Wed, 01 Jan 2024 00:00:00 GMT", "source"=>"Source Name"}, ...]
+# => [
+# {
+#   :title =>    "Title of the news article", 
+#   :link =>     "https://link.to/the/article", 
+#   :pub_date => Time("Wed, 01 Jan 2024 00:00:00 GMT"), 
+#   :descriptions => [
+#     {
+#       :title => "First article, like :title",
+#       :link => "https://link.to/the/first/article",
+#       :author => "Author Name",
+#     },
+#    {
+#      :title => "Second article, same topic, similiar to first",
+#      :link => "https://link.to/the/second/article",
+#     :author => "Another Author Name",
+#    },
+#    ...
+#  ],
+# ]
 ```
 
 

data/Rakefile

@@ -9,9 +9,25 @@ task :default do
 end
 
 task :console do
-  require_relative '../lib/google_news'
-  require 'irb'
-  IRB.start(__FILE__)
+  require_relative 'lib/google_news'
+  require 'pry'
+  Pry.start
 end
 
+namespace :example do
+  task :headlines do
+    require_relative 'lib/google_news'
+    headlines = GoogleNews.headlines(n: 10, language: 'de', country: 'de')
+    headlines.each do |item|
+      puts "Title: #{item[:title]}"
+      puts "Link: #{item[:link]}"
+      puts "Published At: #{item[:pub_date].class}"
+      puts "Descriptions: #{item[:descriptions]}"
+      puts "-" * 40
+    end
+  end
+
+end
+
+
 RSpec::Core::RakeTask.new(:spec)

data/bin/console

@@ -1,14 +1,13 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
-require_relative '../lib/google_news'
-
+require 'google_news'
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require 'pry'
-# Pry.start
+require "pry"
+Pry.start
 
-require 'irb'
-IRB.start(__FILE__)
+#require "irb"
+#IRB.start(__FILE__)

data/bin/google_news

@@ -1,31 +1,33 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/google_news'
+require_relative '../lib/google_news/cli'
 
-arg = ARGV[0]
 
-if %w[help -h --help].include?(arg) || arg.nil? || arg == ''
-  puts <<-TEXT
-    
-    google_news #{GoogleNews::VERSION}
-    https://github.com/magynhard/google_news
+GoogleNews::CLI.run
 
-    Usage:
-      google_news [options]
-    Options:
-      -h, --help          Show this help message
-      -v, --version       Show version
-      -n, --number N      Number of headlines to fetch (default: 10)
-      -c, --country CC    Country code (default: "us")
-      -l, --language LL   Language code (default: "en")
-      --headlines        Fetch top headlines
-      --topic TOPIC      Fetch headlines for a specific topic (WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SPORTS, SCIENCE, HEALTH)
-      --geo LOCATION     Fetch headlines for a specific geographic location (e.g. "48.8566,2.3522" for Paris)
-      --search QUERY     Search headlines for a specific query
-      --website SITE    Fetch headlines from a specific website (e.g. "bbc.com")
-
-  TEXT
-elsif %w[]
-else
-  puts "Invalid command or option. Use --help for usage information."
-end
+#
+# if %w[help -h --help].include?(arg) || arg.nil? || arg == ''
+#   puts <<-TEXT
+#
+#     google_news #{GoogleNews::VERSION}
+#     https://github.com/magynhard/google_news
+#
+#     Usage:
+#       google_news [options]
+#     Options:
+#       -h, --help          Show this help message
+#       -v, --version       Show version
+#       -n, --number N      Number of headlines to fetch (default: 10)
+#       -c, --country CC    Country code (default: "us")
+#       -l, --language LL   Language code (default: "en")
+#       --headlines        Fetch top headlines
+#       --topic TOPIC      Fetch headlines for a specific topic (WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SPORTS, SCIENCE, HEALTH)
+#       --geo LOCATION     Fetch headlines for a specific geographic location (e.g. "48.8566,2.3522" for Paris)
+#       --search QUERY     Search headlines for a specific query
+#       --website SITE    Fetch headlines from a specific website (e.g. "bbc.com")
+#
+#   TEXT
+# elsif %w[]
+# else
+#   puts "Invalid command or option. Use --help for usage information."
+# end

data/google_news.gemspec

@@ -4,24 +4,21 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'google_news/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "google_news"
-  spec.version       = GoogleNews::VERSION
-  spec.executables   = %w[google_news]
-  spec.authors       = ["Matthäus J. N. Beyrle"]
-  spec.email         = ["google_news.gemspec@mail.magynhard.de"]
+  spec.name = "google_news"
+  spec.version = GoogleNews::VERSION
+  spec.executables = %w[google_news]
+  spec.authors = ["Matthäus J. N. Beyrle"]
+  spec.email = ["google_news.gemspec@mail.magynhard.de"]
 
-  spec.summary       = %q{Get Google News headlines from Ruby or the command line.}
-  spec.homepage      = "https://github.com/magynhard/google_news"
-  spec.license       = "MIT"
+  spec.summary = %q{Get Google News headlines from Ruby or the command line.}
+  spec.homepage = "https://github.com/magynhard/google_news"
+  spec.license = "MIT"
 
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = "https://rubygems.org"
-  else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
-  end
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = "#{spec.homepage}/CHANGELOG.md"
 
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -29,11 +26,14 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'rss',      '>= 0.2.6'
+  spec.add_runtime_dependency 'rss', '>= 0.2.6'
   spec.add_runtime_dependency 'open-uri', '>= 0.1.0'
 
-  spec.add_development_dependency 'bundler',  '>= 2.0'
-  spec.add_development_dependency 'rake',     '>= 10.0'
-  spec.add_development_dependency 'rspec',    '>= 3.0'
-  spec.add_development_dependency 'pry',     '>= 0.10.0'
+  spec.add_development_dependency 'bundler', '>= 2.7.1'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry', '~> 0.15.2'
+  spec.add_development_dependency 'fiddle', '~> 1.1.8'
+  spec.add_development_dependency 'ostruct', '~> 0.6.3'
+  spec.add_development_dependency 'nokogiri', '~> 1.18'
 end

data/lib/google_news/cli.rb

@@ -0,0 +1,65 @@
+require 'optparse'
+require 'json'
+require_relative '../google_news'
+
+module GoogleNews
+  #
+  # Command-line interface for GoogleNews
+  #
+  class CLI
+    def self.run(args = ARGV)
+      options = { n: 10, language: 'en' }
+      opt_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: google_news [options]"
+        opts.on('-tTOPIC', '--topic=TOPIC', 'Get news for a specific topic. One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SPORTS, SCIENCE, HEALTH') do |topic|
+          options[:topic] = topic
+        end
+        opts.on('-gPOSITION', '--geo=POSITION', 'Get news for a specific geographic location, e.g. "48.8566,2.3522" for Paris') do |position|
+          options[:geo] = position
+        end
+        opts.on('-cCOUNTRY', '--country=COUNTRY', 'Country code, e.g. "us" for United States (default: us)') do |country|
+          options[:country] = country
+        end
+        opts.on('-lLANGUAGE', '--language=LANGUAGE', 'Language code, e.g. "en" for English (default: en)') do |language|
+          options[:language] = language
+        end
+        opts.on('-nN', '--n=N', Integer, 'Number of headlines to fetch (default: 10)') do |n|
+          options[:n] = n
+        end
+        opts.on('-h', '--help', 'Show this help message') do
+          puts opts
+          exit
+        end
+      end
+      begin
+        opt_parser.parse!(args)
+      rescue OptionParser::InvalidOption => e
+        puts e.message
+        puts opt_parser
+        exit 1
+      end
+      show_logo
+      begin
+        if options[:headlines]
+          headlines = GoogleNews.headlines(country: options[:country] || 'us', language: options[:language] || 'en', n: options[:n] || 10)
+        elsif options[:topic]
+          headlines = GoogleNews.topic(options[:topic], country: options[:country] || 'us', language: options[:language] || 'en', n: options[:n] || 10)
+        elsif options[:geo]
+          headlines = GoogleNews.geo(options[:geo], country: options[:country] || 'us', language: options[:language] || 'en', n: options[:n] || 10)
+        else
+          headlines = GoogleNews.headlines(country: options[:country] || 'us', language: options[:language] || 'en', n: options[:n] || 10)
+        end
+      end
+      puts headlines.to_json
+    end
+
+    def self.show_logo
+      puts
+      puts "GoogleNews CLI v#{GoogleNews::VERSION} - Get Google News headlines from Ruby or the command line."
+      puts ""
+      puts "Powered by the 'google_news' Ruby gem"
+      puts ""
+    end
+
+  end
+end

data/lib/google_news/version.rb

@@ -1,3 +1,3 @@
 module GoogleNews
-  VERSION = '0.0.1'.freeze
+  VERSION = '0.1.0'.freeze
 end

data/lib/google_news.rb

@@ -5,12 +5,15 @@
 require 'open-uri'
 require 'rss'
 require 'uri'
+require 'nokogiri'
+
+require_relative 'google_news/version'
 
 module GoogleNews
   HEADLINES_RSS = 'https://news.google.com/news/rss'.freeze
-  TOPICS_RSS    = 'https://news.google.com/news/rss/headlines/section/topic/'.freeze
-  GEO_RSS       = 'https://news.google.com/news/rss/headlines/section/geo/'.freeze
-  SEARCH_RSS    = 'https://news.google.com/rss/search?q='.freeze
+  TOPICS_RSS = 'https://news.google.com/news/rss/headlines/section/topic/'.freeze
+  GEO_RSS = 'https://news.google.com/news/rss/headlines/section/geo/'.freeze
+  SEARCH_RSS = 'https://news.google.com/rss/search?q='.freeze
 
   TOPICS = %w[WORLD NATION BUSINESS TECHNOLOGY ENTERTAINMENT SPORTS SCIENCE HEALTH].freeze
 
@@ -48,7 +51,7 @@ module GoogleNews
   #
   # Get top headlines for a specific geographic location from Google News
   #
-  # @param [String] position e.g. "48.8566,2.3522" for Paris
+  # @param [String] position name e.g. "Berlin" for Berlin
   # @param [String] country
   # @param [String] language
   # @param [Integer] n
@@ -199,17 +202,18 @@ module GoogleNews
   # Convert an RSS item to a hash
   #
   # @param [RSS::Rss::Channel::Item] item
-  # @return [Hash] with keys :title, :link, :pub_date, :description, and :raw_item
+  # @return [Hash] with keys :title, :link, :pub_date, :descriptions [:title, :link, :author], and :raw_item
   #
   def self.item_to_hash(item)
     {
       title: item.title,
       link: extract_link(item),
       pub_date: (item.respond_to?(:pubDate) ? item.pubDate : nil),
-      description: (item.respond_to?(:description) ? item.description : nil),
+      descriptions: (item.respond_to?(:description) ? parse_description(item.description) : nil),
       raw_item: item
     }
   end
+
   #
   # Extract the link from an RSS item
   #
@@ -223,6 +227,42 @@ module GoogleNews
     nil
   end
 
+  #
+  # Description can consist of a <a> tag with a link, containing the title, followed by a <font> tag with the authors name.
+  #
+  # But it can also consist of a bunch of news, inside a <ol> list, containing <li> items with a <a> tag, followed by a <font> tag each.
+  #
+  # This method creates a array of hashes with :title, :link and :author keys for each news item found in the description.
+  #
+  # @param [String] description as HTML
+  # @return [Array<Hash>] array of news items with :title, :link and :author keys
+  #
+  def self.parse_description(description)
+    return nil if description.nil?
+    doc = Nokogiri::HTML(description)
+    news_items = []
+    if doc.at_css('ol')
+      doc.css('ol li').each do |li|
+        a_tag = li.at_css('a')
+        font_tag = li.at_css('font')
+        news_items << {
+          title: a_tag ? a_tag.text : nil,
+          link: a_tag ? a_tag['href'] : nil,
+          author: font_tag ? font_tag.text : nil
+        }
+      end
+    else
+      a_tag = doc.at_css('a')
+      font_tag = doc.at_css('font')
+      news_items << {
+        title: a_tag ? a_tag.text : nil,
+        link: a_tag ? a_tag['href'] : nil,
+        author: font_tag ? font_tag.text : nil
+      }
+    end
+    news_items
+  end
+
   #
   # Default User-Agent string for HTTP requests
   #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google_news
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Matthäus J. N. Beyrle
@@ -43,56 +43,98 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.7.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.7.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.15.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.2
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.8
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.6.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.3
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
 email:
 - google_news.gemspec@mail.magynhard.de
 executables:
@@ -103,6 +145,9 @@ files:
 - ".gitattributes"
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE
@@ -115,12 +160,16 @@ files:
 - google_news.gemspec
 - lib/custom_errors/invalid_key_type_error.rb
 - lib/google_news.rb
+- lib/google_news/cli.rb
 - lib/google_news/version.rb
 homepage: https://github.com/magynhard/google_news
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/magynhard/google_news
+  source_code_uri: https://github.com/magynhard/google_news
+  changelog_uri: https://github.com/magynhard/google_news/CHANGELOG.md
 rdoc_options: []
 require_paths:
 - lib
@@ -128,14 +177,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.1
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Get Google News headlines from Ruby or the command line.
 test_files: []