history_scraper 1.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3246785d0b76adc995bf2b8f68690f70eb82bb4f
+  data.tar.gz: d58132b79601c8132d2daaede729462b1aade7da
+SHA512:
+  metadata.gz: 6ec98869c9822ca7e1da3db137ddf13b455d535d63ae8798ff060d20b326f0e0e58ccb1f2702ea30f36661614359ecb74b551fc5b6509a4d6417533acc29213d
+  data.tar.gz: 231add83bd0beae6abb2ebecc3dd48dcc4824f48d3878f9c0370542579d6518efcee8f3e02c1745689363fe18b6c5657ffcde4f70fe9fccaf858d753c2489fa7

data/lib/history_scraper.rb

@@ -0,0 +1,66 @@
+require 'open-uri'
+require 'nokogiri'
+require 'date'
+require 'json'
+
+def scrap_year(output_dir)
+  result = {}
+  (1..12).each do |month_index|
+    (1..31).each do |day_index|
+      begin
+        day, month = form_date(day_index, month_index)
+
+        puts "Scraping #{month} #{day}..."
+
+        description, events, births, deaths = extract_from(day, month)
+
+        result["#{month}-#{day}".to_sym] = {
+          description: description, events: events, births: births, deaths: deaths
+        }
+      rescue NoMethodError
+        puts 'It seems this date does not have any episodes.'
+      end
+    end
+  end
+
+  export_to_file(result, output_dir)
+end
+
+private
+
+def form_date(day_index, month_index)
+  date = Date._strptime("#{day_index}/#{month_index}", '%d/%m')
+  [date[:mday], Date::MONTHNAMES[date[:mon]]]
+end
+
+def extract_from(day, month)
+  html = Nokogiri::HTML open("https://en.wikipedia.org/wiki/#{month}_#{day}")
+
+  description = html.css('#mw-content-text p')
+                    .map(&:text)
+                    .find { |text| text.include?("#{month} #{day}") }
+
+  events = parse_ul html.css('#Events')[0].parent.next_element
+  births = parse_ul html.css('#Births')[0].parent.next_element
+  deaths = parse_ul html.css('#Deaths')[0].parent.next_element
+
+  [description, events.compact, births.compact, deaths.compact]
+end
+
+def parse_ul(ul)
+  ul.css('li').map do |li|
+    year, *text = li.text.split(' – ')
+
+    next unless year && !text.empty?
+    { year: year, data: text.join(' – '), kw: parse_keywords(li) }
+  end
+end
+
+def parse_keywords(li)
+  li.css('a').map { |a| { title: a['title'], href: a['href'] } }
+end
+
+def export_to_file(hash_data, dir)
+  File.write(dir, hash_data.to_json)
+  puts "Results stored in #{dir}"
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: history_scraper
+version: !ruby/object:Gem::Version
+  version: 1.0.6
+platform: ruby
+authors:
+- Sebastián Salata
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: Scraps events, births and deaths that occured during a specific day of
+  history.
+email: sa.salatart@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/history_scraper.rb
+homepage: https://github.com/sasalatart/history-scraper-rb
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Wikipedia history scraper.
+test_files: []