webwatchr 0.0.2

data/tests/infra_test.rb

@@ -0,0 +1,271 @@
+#!/usr/bin/ruby
+require "fileutils"
+require "logger"
+require_relative "../lib/webwatchr/site"
+require_relative "./helpers"
+require "tmpdir"
+require "test/unit"
+
+class BaseWebrickTest < Test::Unit::TestCase
+  require "webrick"
+
+  TEST_CONFIG = {
+    wwwroot: File.join(File.dirname(__FILE__), "www-root"),
+    wwwport: 8001,
+    content_is_string_file: "content_is_string.html",
+    content_is_array_file: "content_is_array.html"
+  }.freeze
+
+  class TestFileHandler < WEBrick::HTTPServlet::FileHandler
+    def do_GET(req, res) # rubocop:disable Metrics/MethodName
+      res.status = 200
+      res["Content-Encoding"] = "utf-8"
+      res["Content-Type"] = "text/html; charset=utf-8"
+      res.body = File.open(File.join(TEST_CONFIG[:wwwroot], req.path)).read()
+    end
+  end
+
+  def restart_webrick()
+    @webrick = WEBrick::HTTPServer.new(
+      AccessLog: [],
+      Logger: WEBrick::Log.new("/dev/null", 7),
+      Port: TEST_CONFIG[:wwwport]
+    )
+    @webrick.mount("/", TestFileHandler, TEST_CONFIG[:wwwroot])
+
+    @serv_thread = Thread.new do
+      @webrick.start
+    end
+  end
+
+  def setup
+    @workdir = Dir.mktmpdir
+    @logger_test_io = StringIO.new()
+    MyLog.instance.configure(@logger_test_io, nil, Logger::DEBUG)
+    restart_webrick()
+  end
+
+  def teardown()
+    @webrick.stop
+    @serv_thread.join
+    [TEST_CONFIG[:content_is_string_file], TEST_CONFIG[:content_is_array_file]].each do |tf|
+      File.open(File.join(TEST_CONFIG[:wwwroot], tf), "w") do |f|
+        f.puts ""
+      end
+    end
+    FileUtils.remove_entry_secure(@workdir)
+  end
+end
+
+class TestSimpleStringSite < BaseWebrickTest
+  class TestStringSite < Site::SimpleString
+    def initialize
+      super()
+      @wait = 3600
+    end
+
+    def get_content()
+      return ResultObject.new(@parsed_content.css("div.content").text)
+    end
+  end
+
+  def test_content_is_string
+    content_html = "👌 👌 👌 💯 💯 💯 💯 -KYop-R11Iqo.mp"
+    whole_html = Site::HTML_HEADER.dup
+    whole_html += "<title>test</title><div class='content'>#{content_html}</div>"
+    File.open(File.join(TEST_CONFIG[:wwwroot], TEST_CONFIG[:content_is_string_file]), "w") do |f|
+      f.write whole_html
+    end
+    url = "http://localhost:#{TEST_CONFIG[:wwwport]}/#{TEST_CONFIG[:content_is_string_file]}"
+    wait = 10 * 60
+
+    c = TestStringSite.new
+    c.url = url
+    a = TestAlerter.new()
+    c.alerters = [a]
+    assert { c.load_state_file() == {} }
+    assert { c.should_update?(-9_999_999_999_999) }
+    assert { c.should_update?((Time.now() - wait + 30).to_i) == false }
+    html = c.fetch_url(url)
+    assert { whole_html == html }
+    assert { c.parse_noko(html).css("title").text == "test" }
+    cache_dir = File.join(@workdir, "cache")
+    last_dir = File.join(@workdir, ".lasts")
+    FileUtils.mkdir_p(cache_dir)
+    FileUtils.mkdir_p(last_dir)
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    assert { c.state_file.end_with?("last-localhost-2182cd5c8685baed48f692ed72d7a89f") }
+    expected_error = "DEBUG -- TestSimpleStringSite::TestStringSite: Alerting new stuff"
+    last_error = @logger_test_io.string.split("\n")[-1]
+    assert { last_error.end_with?(expected_error) }
+    first_pass_content = Site::HTML_HEADER + content_html
+    assert { c.content.to_html == content_html }
+    assert { c.generate_html_content == first_pass_content }
+    assert { a.result == c.content }
+
+    File.open(File.join(TEST_CONFIG[:wwwroot], TEST_CONFIG[:content_is_string_file]), "w+") do |f|
+      f.write whole_html.gsub("</div>", " new ! </div>")
+    end
+    c = TestStringSite.new
+    c.url = url
+    cache_dir = File.join(@workdir, "cache")
+    last_dir = File.join(@workdir, ".lasts")
+    FileUtils.mkdir_p(cache_dir)
+    FileUtils.mkdir_p(last_dir)
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    c.comment = "lol"
+    expected_error = "INFO -- TestSimpleStringSite::TestStringSite: Too soon to update #{url}"
+    last_error = @logger_test_io.string.split("\n")[-1]
+    assert { last_error.end_with?(expected_error) }
+    assert { c.content.nil? }
+    assert { c.generate_html_content.nil? }
+    assert { c.name == url }
+
+    c.every = 0
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    expected_error = "DEBUG -- TestSimpleStringSite::TestStringSite: Alerting new stuff"
+    last_error = @logger_test_io.string.split("\n")[-1]
+    assert { last_error.end_with?(expected_error) }
+    assert { c.content.to_html == "#{content_html} new ! " }
+    assert { c.generate_html_content == "#{first_pass_content} new ! " }
+    assert { c.name == url }
+    result_last = JSON.parse(File.read(c.state_file), create_additions: true)
+    result_last.delete("time")
+    assert { result_last["url"] == url }
+    assert { result_last["content"].message == "#{content_html} new ! " }
+    assert { result_last["wait"] == 0 }
+  end
+end
+
+class TestArraySites < BaseWebrickTest
+  class TestArraySite < Site::Articles
+    def initialize
+      super()
+      @wait = 3600
+    end
+
+    def get_content()
+      res = []
+      @parsed_content.css("div").each do |x|
+        a, b = x.text.split("-").map(&:strip)
+        add_article({ "id" => a, "url" => a, "title" => b })
+      end
+      return res
+    end
+  end
+
+  def test_content_is_array
+    whole_html = "#{Site::HTML_HEADER.dup}<title>test</title>👌 👌 👌 💯 💯 💯 💯 -KYop-R11Iqo.mp4<div> lol - lilo</div> <div> fi - fu</div>"
+    File.open(File.join(TEST_CONFIG[:wwwroot], TEST_CONFIG[:content_is_array_file]), "w") do |f|
+      f.write whole_html
+    end
+    url = "http://localhost:#{TEST_CONFIG[:wwwport]}/#{TEST_CONFIG[:content_is_array_file]}"
+    wait = 10 * 60
+
+    c = TestArraySite.new
+    c.url = url
+    a = TestAlerter.new()
+    c.alerters = [a]
+    assert { c.load_state_file() == {} }
+    assert { c.should_update?(-9_999_999_999_999) }
+    assert { !c.should_update?((Time.now() - wait + 30).to_i) }
+    html = c.fetch_url(url)
+    assert { html == whole_html }
+    assert { c.parse_noko(html).css("title").text == "test" }
+
+    # First full run, Get 2 things
+    cache_dir = File.join(@workdir, "cache")
+    last_dir = File.join(@workdir, ".lasts")
+    FileUtils.mkdir_p(cache_dir)
+    FileUtils.mkdir_p(last_dir)
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    assert { c.state_file.end_with?("last-localhost-35e711989b197f20f3d4936e91a2c079") }
+    expected_error = "DEBUG -- TestArraySites::TestArraySite: Alerting new stuff"
+    last_error = @logger_test_io.string.split("\n")[-1].strip()
+    assert { last_error.end_with?(expected_error) }
+    expected_html = Site::HTML_HEADER.dup + [
+      "<ul style='list-style-type: none;'>",
+      "<li id='lol'><a href='lol'>lilo</a></li>",
+      "<li id='fi'><a href='fi'>fu</a></li>",
+      "</ul>"
+    ].join("\n")
+    c.content.each { |x| x.delete('_timestamp') }
+    assert {
+      c.content == [
+        { "id" => "lol", "url" => "lol", "title" => "lilo" },
+        { "id" => "fi", "url" => "fi", "title" => "fu" }
+      ]
+    }
+    assert { c.generate_html_content == expected_html }
+
+    result = ""
+
+    File.open(File.join(TEST_CONFIG[:wwwroot], TEST_CONFIG[:content_is_array_file]), "a+") do |f|
+      f.write "<div>new! - new </div>"
+    end
+    c = TestArraySite.new
+    c.url = url
+    a = TestAlerter.new()
+    c.alerters = [a]
+    # Second run don't d anything because we shouldn't rerun
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    expected_error = "INFO -- TestArraySites::TestArraySite: Too soon to update #{url}"
+    last_error = @logger_test_io.string.split("\n")[-1]
+    assert { last_error.end_with?(expected_error) }
+    assert { result == "" }
+
+    result = ""
+
+    c.content.each { |x| x.delete('_timestamp') }
+
+    c.every = 0
+    # This time we set new things, and wait is 0 so we are good to go
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    expected_error = "DEBUG -- TestArraySites::TestArraySite: Alerting new stuff"
+    last_error = @logger_test_io.string.split("\n")[-1]
+    assert { last_error.end_with?(expected_error) }
+    expected_html = Site::HTML_HEADER.dup + [
+      "<ul style='list-style-type: none;'>",
+      "<li id='new!'><a href='new!'>new</a></li>",
+      "</ul>"
+    ].join("\n")
+
+    c.content.each { |x| x.delete('_timestamp') }
+    assert { c.content == [{ "id" => "new!", "url" => "new!", "title" => "new" }] }
+    assert { c.generate_html_content == expected_html }
+    expected_last = { "url" => "http://localhost:#{TEST_CONFIG[:wwwport]}/#{TEST_CONFIG[:content_is_array_file]}",
+                      "previous_content" => [{ "id" => "lol", "url" => "lol", "title" => "lilo" },
+                                             { "id" => "fi", "url" => "fi", "title" => "fu" }],
+                      "wait" => 0,
+                      "content" => [{ "id" => "lol", "title" => "lilo", "url" => "lol" },
+                                    { "id" => "fi", "title" => "fu", "url" => "fi" },
+                                    { "id" => "new!", "title" => "new", "url" => "new!" }] }
+    result_last = JSON.parse(File.read(c.state_file))
+    result_last.delete("time")
+    result_last["content"].each do |item|
+      item.delete("_timestamp")
+    end
+    result_last["previous_content"].each do |item|
+      item.delete("_timestamp")
+    end
+    assert { expected_last == result_last }
+
+    result = ""
+
+    c = TestArraySite.new
+    c.url = url
+    a = TestAlerter.new()
+    c.alerters = [a]
+    c.every = 0
+    # Now, we don't call the alert Proc because we have no new things
+    c.update(cache_dir: cache_dir, last_dir: last_dir)
+    expected_error = "INFO -- TestArraySites::TestArraySite: Nothing new for #{url}"
+    last_error = @logger_test_io.string.split("\n")[-1]
+    assert { last_error.end_with?(expected_error) }
+    expected_html = Site::HTML_HEADER.dup + [
+      "<ul style=\"list-style-type: none;\">",
+      "</ul>"
+    ].join("\n")
+    assert { result == "" }
+  end
+end

data/webwatchr.gemspec

@@ -0,0 +1,12 @@
+Gem::Specification.new do |s|
+  s.name        = "webwatchr"
+  s.version     = "0.0.2"
+  s.summary     = "Scrapes stuff and tells you of updates."
+  s.description = "Scrapes stuff and tells you of updates. Exciting!"
+  s.authors     = ["Renzo"]
+  s.email       = "webwatchr-gem@renzokuken.eu"
+  s.files       = Dir['README.md', 'tests/**.*', 'lib/**/*', 'Rakefile', 'Gemfile', 'LICENSE', 'webwatchr.gemspec']
+  s.homepage    = "https://github.com/conchyliculture/webwatchr"
+  s.license     = "GPL-3.0-or-later"
+  s.required_ruby_version = ">=2.5.0"
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: webwatchr
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Renzo
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Scrapes stuff and tells you of updates. Exciting!
+email: webwatchr-gem@renzokuken.eu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/sites/bandcamp.rb
+- lib/sites/bsky.rb
+- lib/sites/postch.rb
+- lib/sites/songkick.rb
+- lib/webwatchr.rb
+- lib/webwatchr/alerting.rb
+- lib/webwatchr/base.rb
+- lib/webwatchr/logger.rb
+- lib/webwatchr/main.rb
+- lib/webwatchr/site.rb
+- tests/helpers.rb
+- tests/infra_test.rb
+- webwatchr.gemspec
+homepage: https://github.com/conchyliculture/webwatchr
+licenses:
+- GPL-3.0-or-later
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Scrapes stuff and tells you of updates.
+test_files: []