microformat_scraper 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 4b44795b4d645cb6953ae2fa501b646666d9829f
4
+ data.tar.gz: 8ad97e266b17e1ea523618802c400b6b3435c8ab
5
+ SHA512:
6
+ metadata.gz: 4988fceee08731ff7f1953e68603b6882206454a72a09366efc5cd94a95fe330ec93bc122a3907111cb99b8fc3c94b51c2c338643341932d265be9210ada01d7
7
+ data.tar.gz: 86e5a16a5ac2b51eb35881ace2527d4373d9ec9702f60988081f9d7125575cd5f5776f66abbeaa07cc865d501d943d52f43db68d3d43190898ee4e8d319ae94e
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.1.2
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in microformat_scraper.gemspec
4
+ gemspec
@@ -0,0 +1,39 @@
1
+ # MicroformatScraper
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/microformat_scraper`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'microformat_scraper'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install microformat_scraper
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ 1. Fork it ( https://github.com/[my-github-username]/microformat_scraper/fork )
36
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
37
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
38
+ 4. Push to the branch (`git push origin my-new-feature`)
39
+ 5. Create a new Pull Request
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new('spec')
5
+
6
+ task :default => :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "microformat_scraper"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'microformat_scraper'
4
+
5
+ url = ARGV.last
6
+ puts "scraping events from #{url}"
7
+
8
+ page = MicroformatScraper::Page.new(url)
9
+ parser = MicroformatScraper::EventParser.new
10
+
11
+ events = parser.parse(page.content)
12
+ puts "found #{events.count} events"
13
+
14
+ events.each do |event|
15
+ puts "START TIME: #{event.start_time}"
16
+ puts "END TIME: #{event.end_time}"
17
+ puts "DESCRIPTION: #{event.description}"
18
+ puts "SUMMARY: #{event.summary}"
19
+ end
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,13 @@
1
+ require "microformat_scraper/version"
2
+ require "microformat_scraper/page"
3
+ require "microformat_scraper/event_parser"
4
+
5
+ module MicroformatScraper
6
+
7
+ def self.scheduled_events(url)
8
+ page = Page.new(url)
9
+ parser = EventParser.new
10
+ parser.parse(page.content)
11
+ end
12
+
13
+ end
@@ -0,0 +1,14 @@
1
+ require 'virtus'
2
+
3
+ module MicroformatScraper
4
+
5
+ class Event
6
+ include Virtus.model
7
+
8
+ attribute :start_time, Time
9
+ attribute :end_time, Time
10
+ attribute :description, String
11
+ attribute :summary, String
12
+ end
13
+
14
+ end
@@ -0,0 +1,44 @@
1
+ require 'microformat_scraper/event'
2
+ require 'nokogiri'
3
+
4
+ module MicroformatScraper
5
+
6
+ class EventParser
7
+
8
+ # This method is extremely fragile, but so far EOLclub.org has been sticking to
9
+ # the same format, so that may be okay.
10
+ def parse(content)
11
+ doc = Nokogiri::HTML.parse(content)
12
+ doc.css('.vevent').map{|elem| parse_event(elem)}
13
+ end
14
+
15
+ private
16
+
17
+ def parse_event(elem)
18
+ summary = mf_value(elem, '.summary')
19
+
20
+ description = elem.css('.description').map(&:to_html).join
21
+
22
+ start_time = mf_value(elem, '.dtstart')
23
+ end_time = mf_value(elem, '.dtend')
24
+
25
+ Event.new(
26
+ start_time: start_time,
27
+ end_time: end_time,
28
+ description: description,
29
+ summary: summary,
30
+ )
31
+ end
32
+
33
+ def mf_value(elem, css_path)
34
+ el = elem.css(css_path).first
35
+ if attr = el.attributes['title']
36
+ attr.value
37
+ else
38
+ el.text
39
+ end
40
+ end
41
+
42
+ end
43
+
44
+ end
@@ -0,0 +1,15 @@
1
+ require 'open-uri'
2
+
3
+ module MicroformatScraper
4
+
5
+ class Page
6
+ def initialize(url)
7
+ @url = url
8
+ end
9
+
10
+ def content
11
+ open(@url).read
12
+ end
13
+ end
14
+
15
+ end
@@ -0,0 +1,3 @@
1
+ module MicroformatScraper
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,28 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'microformat_scraper/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "microformat_scraper"
8
+ spec.version = MicroformatScraper::VERSION
9
+ spec.authors = ["Matt Gillooly"]
10
+ spec.email = ["git@mattgillooly.com"]
11
+
12
+ spec.summary = %q{Microformat event scraper}
13
+ spec.description = %q{Scrape scheduled events from microformatted web pages}
14
+ spec.homepage = "http://pvdtechevents.com/"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = "exe"
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency 'nokogiri'
22
+ spec.add_dependency 'virtus', '~> 1.0'
23
+ spec.add_development_dependency "bundler", "~> 1.9"
24
+ spec.add_development_dependency "rake", "~> 10.0"
25
+ spec.add_development_dependency 'rspec'
26
+ spec.add_development_dependency 'vcr'
27
+ spec.add_development_dependency 'webmock'
28
+ end
metadata ADDED
@@ -0,0 +1,156 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: microformat_scraper
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Matt Gillooly
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2015-09-21 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: nokogiri
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: virtus
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.9'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.9'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rspec
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: vcr
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: webmock
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ description: Scrape scheduled events from microformatted web pages
112
+ email:
113
+ - git@mattgillooly.com
114
+ executables: []
115
+ extensions: []
116
+ extra_rdoc_files: []
117
+ files:
118
+ - ".gitignore"
119
+ - ".rspec"
120
+ - ".travis.yml"
121
+ - Gemfile
122
+ - README.md
123
+ - Rakefile
124
+ - bin/console
125
+ - bin/scrape_event
126
+ - bin/setup
127
+ - lib/microformat_scraper.rb
128
+ - lib/microformat_scraper/event.rb
129
+ - lib/microformat_scraper/event_parser.rb
130
+ - lib/microformat_scraper/page.rb
131
+ - lib/microformat_scraper/version.rb
132
+ - microformat_scraper.gemspec
133
+ homepage: http://pvdtechevents.com/
134
+ licenses: []
135
+ metadata: {}
136
+ post_install_message:
137
+ rdoc_options: []
138
+ require_paths:
139
+ - lib
140
+ required_ruby_version: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - ">="
143
+ - !ruby/object:Gem::Version
144
+ version: '0'
145
+ required_rubygems_version: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - ">="
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ requirements: []
151
+ rubyforge_project:
152
+ rubygems_version: 2.2.2
153
+ signing_key:
154
+ specification_version: 4
155
+ summary: Microformat event scraper
156
+ test_files: []