diff_news 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b8fb36e0227389624d87173d0c0c6a1345a7f933
+  data.tar.gz: 4fa5f28152f01432fdde5b51df1774439080024c
+SHA512:
+  metadata.gz: adca1f78c3bcf8fb43eab5b88fbe87117d98f4e56e30f577ab3d438dba9d78e8e5bcc8bce8092e948bcf3dca0515d379492bf6eac59b9ff7f1fffe62f54c7c2e
+  data.tar.gz: 357f4e1087332e2ff2618ea76fa427c2e732e8cffe6e22f9743dbb94683b9456f76fb96c7abe19bcd91dee361fa3fc4529b1b8ec5d3018df212ac5e63156357a

data/lib/default_history.rb

@@ -0,0 +1,42 @@
+#
+# Store and retrieve a history by provider and uuid
+#
+
+require 'revision'
+
+module DiffNews
+
+  class DefaultHistory
+    attr_reader :revisions
+
+    def initialize provider, uuid
+      @provider = provider
+      @uuid = uuid
+
+      @revisions = []
+    end
+  
+    def load
+    end
+
+    def store
+    end
+
+    def revision_changed? rev1, rev2
+      return true if rev1 == nil
+      return rev1.title != rev2.title || rev1.teaser != rev2.teaser || rev1.text != rev2.text
+    end
+
+    def append_changed_revision title, teaser, text
+      new_revision = Revision.new @provider, @uuid, @revisions.count, title, teaser, text
+      if revision_changed? @revisions.last, new_revision
+        @revisions << new_revision
+      end
+    end
+
+    def clear
+      @revisions.clear
+    end
+  end
+
+end

data/lib/diff_news.rb

@@ -0,0 +1,12 @@
+#
+# Diff News
+#
+
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'service'
+require 'default_history'
+require 'file_history'
+require 'news_service/tagesanzeiger'
+require 'news_service/nzz'
+require 'news_service/mock'

data/lib/file_history.rb

@@ -0,0 +1,36 @@
+#
+# Store and retrieve a history by provider and uuid
+#
+
+require 'revision'
+require 'yaml'
+
+module DiffNews
+
+  class FileHistory < DefaultHistory
+    attr_reader :revisions
+
+    def initialize provider, uuid
+      super provider, uuid
+    end
+
+    def file_path(extension)
+       "#{@provider}/#{@uuid}.#{extension}"
+    end
+  
+    def load
+      if File.exist? file_path('yml')
+        @revisions = YAML.load(File.open(file_path('yml'), 'r:UTF-8', &:read))
+      end
+    end
+
+    def store
+      if @revisions
+        File.open(file_path('yml'), 'w:UTF-8') do |file|
+          file.write(YAML::dump(@revisions))
+        end
+      end
+    end
+  end
+
+end

data/lib/news_service/mock.rb

@@ -0,0 +1,35 @@
+#
+# Mock Service Implementation
+#
+
+require 'service'
+
+module DiffNews
+
+  class MockService < Service
+    attr_reader :title, :teaser, :text
+
+    def initialize url
+      @title = "Mock Title"
+      @teaser = "Mock Teaser"
+      @text = "Mock Text"
+
+      super url
+    end
+
+    def self.provider
+      'mock_service'
+    end
+
+    def self.valid_url
+      /mock_me/
+    end
+   
+    def load_document
+    end
+
+    register_service
+  end
+
+end
+

data/lib/news_service/nzz.rb

@@ -0,0 +1,46 @@
+require 'service'
+
+module DiffNews
+
+  class NZZ < Service
+    def initialize url
+      super url
+    end
+  
+    def self.provider
+      'nzz.ch'
+    end
+
+    def self.valid_url
+      /^https?:\/\/(www.)?nzz.ch/
+    end
+
+  private   
+    def title
+      normalize_text(strip_html(@doc.css(".title__name").first))
+    rescue
+      ''
+    end
+  
+    def teaser
+      normalize_text(strip_html(@doc.css(".leadtext").first))
+    rescue
+      ''
+    end
+  
+    def text
+      content = @doc.css("article.content").first
+    
+      content.css('script').remove
+      content.css('.inlinePoll').remove
+    
+      normalize_text(strip_html(content))
+    rescue
+      ''
+    end
+
+    register_service
+  end
+
+end
+

data/lib/news_service/tagesanzeiger.rb

@@ -0,0 +1,56 @@
+require 'service'
+require 'revision'
+
+module DiffNews
+
+  class Tagesanzeiger < Service
+    def initialize url
+      super normalize_url(url)
+    end
+
+    def self.provider
+      'tagesanzeiger.ch'
+    end
+
+    def self.valid_url
+      /^https?:\/\/(www.)?tagesanzeiger.ch/
+    end
+
+private
+
+  # Remove the SEO parts, making the URL more unique
+    def normalize_url url
+      parts = url.split('/')
+      if parts.count > 5
+        (parts.first(4) + parts.last(2)).join('/')
+      else
+        url
+      end
+    end
+
+    def title
+      normalize_text(strip_html(@doc.css("h1").first))
+    rescue
+      ''
+    end
+  
+    def teaser
+      normalize_text(strip_html(@doc.css("h3").first))
+    rescue
+      ''
+    end
+  
+    def text
+      content = @doc.css("#mainContent").first
+    
+      content.css('script').remove
+      content.css('.inlinePoll').remove
+    
+      normalize_text(strip_html(content))
+    rescue
+      ''
+    end
+
+    register_service
+  end
+end

data/lib/revision.rb

@@ -0,0 +1,7 @@
+#
+# POD for revision, container for the revision data stored in file or db
+#
+
+module DiffNews
+  Revision = Struct.new("Revision", :provider, :uuid, :revision, :title, :teaser, :text)
+end

data/lib/service.rb

@@ -0,0 +1,86 @@
+#
+# News Service Factory
+#
+
+require 'uuidtools'
+require 'nokogiri'
+require 'open-uri'
+
+require 'default_history'
+
+module DiffNews
+
+  class UnknownNewspageError < StandardError
+  end
+
+  class Service
+    attr_reader :history
+    @@services = []
+    @@store_class = DefaultHistory
+
+    def provider
+      self.class.provider
+    end
+  
+    def current_revision
+      return @history.revisions.last
+    end
+
+    def self.store_class= store_class
+      @@store_class = store_class
+    end
+
+    def initialize url
+      @url = url
+      @uuid = UUIDTools::UUID.sha1_create(UUIDTools::UUID_URL_NAMESPACE, url)
+      load_document
+      load_history
+    end
+  
+    def self.register_service
+      puts "Registered Service Provider: #{self.provider}"
+      @@services << self
+    end
+
+    def self.create url
+      match = @@services.select { |s| url =~ s.valid_url }
+      raise UnknownNewspageError if match.none?
+
+      match.first.new url
+    end
+
+    def strip_html partial
+      partial.css("br").each { |node| node.replace("\n") }
+      partial.css("p").each { |node| node.replace("#{node.text.strip}\n\n") }
+      partial.text.strip
+    end
+
+    def normalize_text text
+      empty_lines = 0
+      result = ''
+      text.each_line do |line|
+        line = line.strip
+        line = line.gsub(/\s+/, ' ')
+        empty_lines = line.length == 0 ? empty_lines + 1 : 0
+        if empty_lines < 2
+          result += line.strip + "\n"
+        end
+      end
+      return result
+    end
+
+  private
+    def load_document
+      @doc = Nokogiri::HTML(open(@url))
+    end
+
+    def load_history
+      @history = @@store_class.new provider, @uuid
+
+      @history.load
+      @history.append_changed_revision title, teaser, text
+      @history.store
+    end
+  end
+
+end

data/lib/version.info

@@ -0,0 +1 @@
+0.9.0

data/lib/version.rb

@@ -0,0 +1,17 @@
+#
+# Version of vcs.rb
+#
+
+module DiffNews
+  def self.version_path
+    File.expand_path("version.info", File.dirname(__FILE__))
+  end
+
+  def self.version
+    File.open(version_path, &:readline)
+  end
+end
+
+$vcs_ruby_version = Gem::Version.new(DiffNews::version)
+$vcs_ruby_name = 'DiffNews makes changes visible'
+$vcs_ruby_short = 'diff_news'

data/test/file_store_test.rb

@@ -0,0 +1,18 @@
+require 'minitest'
+require 'diff_news'
+
+class FileStoreTest < Minitest::Test
+
+  def mock_service_file_history
+    DiffNews::Service::store_class = DiffNews::FileHistory
+    mock = DiffNews::Service.create 'mock_me'
+  end
+
+  def tagi_history
+    DiffNews::Service::store_class = DiffNews::FileHistory
+    tagi = DiffNews::Service.create 'http://www.tagesanzeiger.ch/sport/tennis/aufschlag-smash-und-tschuess/story/10062709'
+
+    assert_equal 1, tagi.history.revisions.count
+  end
+
+end

data/test/news_service_test.rb

@@ -0,0 +1,26 @@
+require 'minitest'
+require 'diff_news'
+
+class NewsServiceTest < Minitest::Test
+
+  def test_mock_service_history
+    DiffNews::Service::store_class = DiffNews::DefaultHistory
+    service = DiffNews::Service.create 'mock_me'
+
+    assert_equal 1, service.history.revisions.count
+    assert_equal "Mock Teaser", service.history.revisions.first.teaser
+    assert_equal "Mock Title", service.current_revision.title
+  end
+
+  def test_tagi_history
+    DiffNews::Service::store_class = DiffNews::DefaultHistory
+    tagi = DiffNews::Service.create 'http://www.tagesanzeiger.ch/sport/tennis/aufschlag-smash-und-tschuess/story/10062709'
+    assert_equal 1, tagi.history.revisions.count
+  end
+
+  def test_nzz_history
+    DiffNews::Service::store_class = DiffNews::DefaultHistory
+    nzz = DiffNews::Service.create 'https://www.nzz.ch/finanzen/abgang-eines-wichtigen-softwarespezialisten-die-tesla-aktie-hat-derzeit-gegenwind-ld.1304770'
+    assert_equal 1, nzz.history.revisions.count
+  end
+end

data/test/service_test.rb

@@ -0,0 +1,16 @@
+require 'minitest'
+require 'diff_news'
+
+class ServiceTest < Minitest::Test
+  def test_create_providers
+    providers = {
+      'http://www.tagesanzeiger.ch/sport/tennis/aufschlag-smash-und-tschuess/story/10062709' => 'tagesanzeiger.ch',
+      'https://www.nzz.ch/international/europa/abstimmung-im-eu-parlament-xxx-fuer-beitrittsverhandlungen-mit-der-tuerkei-ld.1304606' => 'nzz.ch'
+    }
+
+    providers.each do |url, provider|
+      service = DiffNews::Service.create url
+      assert_equal service.provider, provider
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+# file: test/test_helper.rb
+require 'minitest/autorun'
+require 'minitest/reporters'
+
+Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: diff_news
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Thomas Bruderer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.0.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: ''
+email: apophis@apophis.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/default_history.rb
+- lib/diff_news.rb
+- lib/file_history.rb
+- lib/news_service/mock.rb
+- lib/news_service/nzz.rb
+- lib/news_service/tagesanzeiger.rb
+- lib/revision.rb
+- lib/service.rb
+- lib/version.info
+- lib/version.rb
+- test/file_store_test.rb
+- test/news_service_test.rb
+- test/service_test.rb
+- test/test_helper.rb
+homepage: https://github.com/FreeApophis/DiffNews
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: Tool to keep track of content changes on news sites
+test_files:
+- test/news_service_test.rb
+- test/file_store_test.rb
+- test/test_helper.rb
+- test/service_test.rb