scrapperd 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 227704fb9f04229706b060e558decb3a22c70ba7
4
+ data.tar.gz: 8f3b6972e1d9c9f0dc0b01b0fc75a03fa438283c
5
+ SHA512:
6
+ metadata.gz: d0c4697138145a3d3fc3f3a1fc189a38e06a0994db1e973f1416f0d7b82002987c9e9d29d5e1ae7201f975fa13ff8e1791d5b06517f02460441e1fde54208fb5
7
+ data.tar.gz: 735e9924e5cff405ec933a4c640b5e9c504172b27766b1b8ac50cf7b05aba410df2891207f41ef2117e03708a041e1f8235aa45b8396837fc1afb271c77bf0d1
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ .ruby-version
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in scrapperd.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Victor
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,54 @@
1
+ # Scrapperd
2
+
3
+ Scrapperd is a very simple scrapper for the public user activity RSS in [Letterboxd](http://letterboxd.com). It relies in OpenStruct to build objects.
4
+
5
+
6
+ ## Installation
7
+
8
+ To get the latest stable:
9
+
10
+ gem 'scrapperd'
11
+
12
+ To get the latest code:
13
+
14
+ gem 'scrapperd', git: https://github.com/eltercero/scrapperd
15
+
16
+ And then execute:
17
+
18
+ $ bundle
19
+
20
+ ## Usage
21
+
22
+ ```ruby
23
+ >> require 'scrapperd'
24
+
25
+ >> client = Scrapperd::Base.new
26
+
27
+ # Example
28
+ #####################################################
29
+ # Return the RSS for a username
30
+ >> activities = client.fetch('eltercero')
31
+ => [#<Scrapperd::Activity ...]
32
+ >> film = activities.first
33
+ => #<Scrapperd::Activity title="Porco Rosso, 1992", score=8, link="http://letterboxd.com/eltercero/film/porco-rosso/", film_link="http://letterboxd.com/film/porco-rosso/", watched_at=2015-03-13 00:00:00 +0100, added_at=2015-03-13 22:52:18 +0100, original_response=#<Hashie::Mash dc_creator="V\xC3\xADctor" description="<p><img src=\"http://zardoz.cf.letterboxd.com/resized/film-poster/4/5/1/3/0/45130-porco-rosso-0-150-0-222-crop.jpg\"/></p> <p>Watched on Friday March 13, 2015.</p>" guid="letterboxd-watch-7901161" link="http://letterboxd.com/eltercero/film/porco-rosso/" pubDate=2015-03-13 22:52:18 +0100 title="Porco Rosso, 1992 - \xE2\x98\x85\xE2\x98\x85\xE2\x98\x85\xE2\x98\x85">>
34
+ >> film.title
35
+ => "Porco Rosso, 1992"
36
+ >> film.score
37
+ => 8
38
+ >> film.watched_at
39
+ => 2015-03-13 00:00:00 +0100
40
+ ```
41
+
42
+ ## Contributing
43
+
44
+ Please, be my guest!
45
+
46
+ 1. Fork it ( http://github.com/eltercero/scrapperd/fork )
47
+ 2. Create your feature branch (`git checkout -b sequels-can-be-good`). Don't forget to test your changes!
48
+ 3. Commit your changes (`git commit -am 'Remember Godfather 2!'`)
49
+ 4. Push to the branch (`git push origin sequels-can-be-good`)
50
+ 5. Create new Pull Request
51
+
52
+ ## License
53
+
54
+ Released under the [MIT License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,8 @@
1
+ require 'rake/testtask'
2
+
3
+ Rake::TestTask.new do |t|
4
+ t.libs << 'test'
5
+ t.pattern = "test/*_test.rb"
6
+ end
7
+
8
+ task :default => :test
@@ -0,0 +1,6 @@
1
+ require 'hashie/mash'
2
+ require 'scrapperd/base'
3
+
4
+ module Scrapperd
5
+
6
+ end
@@ -0,0 +1,41 @@
1
+ require 'simple-rss'
2
+ require 'ostruct'
3
+ require 'pry'
4
+
5
+ module Scrapperd
6
+ class Activity < ::OpenStruct
7
+
8
+ def initialize(item)
9
+ super(Hashie::Mash.new({
10
+ title: title_from_title(item[:title]),
11
+ score: score_from_title(item[:title]),
12
+ link: item[:link].force_encoding("UTF-8"),
13
+ film_link: "http://letterboxd.com/film/#{nicetitle_from_url(item[:link])}/".force_encoding("UTF-8"),
14
+ watched_at: watched_at_from_description(item[:description]),
15
+ added_at: item[:pubDate],
16
+ original_response: item
17
+ })
18
+ )
19
+ end
20
+
21
+ def nicetitle_from_url(url)
22
+ url.split('/').last
23
+ end
24
+
25
+ def watched_at_from_description(description)
26
+ string_date = description.match(/<p>Watched on (.*)<\/p>/)[1]
27
+ Time.parse string_date
28
+ end
29
+
30
+ def title_from_title(title)
31
+ title.split(' - ')[0..-2].join(' - ').force_encoding("UTF-8")
32
+ end
33
+
34
+ def score_from_title(title)
35
+ html_score = title.split(' - ').last
36
+ html_score.force_encoding("UTF-8")
37
+ html_score.split(//).map{|ind_score| ind_score == "★" ? 2 : 1 }.inject{|sum,x| sum + x }
38
+ end
39
+
40
+ end
41
+ end
@@ -0,0 +1,21 @@
1
+ require 'simple-rss'
2
+ require 'open-uri'
3
+ require 'pry'
4
+
5
+ require 'scrapperd/activity'
6
+ require 'scrapperd/error'
7
+
8
+ module Scrapperd
9
+ class Base
10
+
11
+ BASE_URL = "http://letterboxd.com/"
12
+
13
+ def fetch(username)
14
+ raise Scrapperd::Error::BadRequest.new('options must be in a Hash') unless username
15
+
16
+ rss = SimpleRSS.parse(open("#{BASE_URL}#{username}/rss"))
17
+ rss.items.map{|item| Scrapperd::Activity.new(item)}
18
+ end
19
+
20
+ end
21
+ end
@@ -0,0 +1,9 @@
1
+ module Scrapperd
2
+ class Error < StandardError
3
+ def initialize(message)
4
+ super(message)
5
+ end
6
+ end
7
+
8
+ class Error::BadRequest < Scrapperd::Error; end
9
+ end
@@ -0,0 +1,3 @@
1
+ module Scrapperd
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'scrapperd/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "scrapperd"
8
+ spec.version = Scrapperd::VERSION
9
+ spec.authors = ["Víctor Martín"]
10
+ spec.email = ["victor.martin84@gmail.com"]
11
+ spec.summary = %q{Scrapper for Letterboxd user's activity}
12
+ spec.description = %q{Scrapper for Letterboxd user's activity using public user RSS}
13
+ spec.homepage = "https://github.com/eltercero/scrapperd"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
+
19
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
20
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
21
+ spec.require_paths = ["lib"]
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.5"
24
+ spec.add_development_dependency "rake", '~> 10.1'
25
+
26
+ spec.add_development_dependency 'minitest', '~>5.2'
27
+ spec.add_dependency 'simple-rss','~> 1.3', '>= 1.3.1'
28
+ spec.add_dependency 'hashie', '~>2.0'
29
+ end
@@ -0,0 +1,21 @@
1
+ require 'test_helper'
2
+
3
+ class ActivityTest < MiniTest::Unit::TestCase
4
+ def test_activity_conversion
5
+ item = { :title => "Nausica\xC3\xA4 of the Valley of the Wind, 1984 - \xE2\x98\x85\xE2\x98\x85\xE2\x98\x85\xC2\xBD",
6
+ :link => "http://letterboxd.com/eltercero/film/nausicaa-of-the-valley-of-the-wind/",
7
+ :description => "<p><img src=\"http://primer.cf.letterboxd.com/resized/film-poster/5/1/9/6/9/51969-nausicaa-of-the-valley-of-the-wind-0-150-0-222-crop.jpg\"/></p> <p>Watched on Saturday March 14, 2015.</p>",
8
+ :pubDate => Time.new('2015-03-14 19:45:50 +0100'),
9
+ :guid => "letterboxd-watch-7911516",
10
+ :dc_creator => "V\xC3\xADctor"}
11
+
12
+ activity = Scrapperd::Activity.new(item)
13
+
14
+ assert_equal activity.title, "Nausica\xC3\xA4 of the Valley of the Wind, 1984"
15
+ assert_equal activity.score, 7
16
+ assert_equal activity.link, "http://letterboxd.com/eltercero/film/nausicaa-of-the-valley-of-the-wind/"
17
+ assert_equal activity.film_link, "http://letterboxd.com/film/nausicaa-of-the-valley-of-the-wind/"
18
+ assert_equal activity.watched_at, Time.parse("Saturday March 14, 2015.")
19
+ assert_equal activity.added_at, Time.new('2015-03-14 19:45:50 +0100')
20
+ end
21
+ end
@@ -0,0 +1,4 @@
1
+ require 'scrapperd'
2
+ require 'minitest/autorun'
3
+ require 'minitest/unit'
4
+ require 'minitest/pride'
metadata ADDED
@@ -0,0 +1,135 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: scrapperd
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Víctor Martín
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-03-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.5'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.5'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.1'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: minitest
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '5.2'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '5.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: simple-rss
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.3'
62
+ - - ">="
63
+ - !ruby/object:Gem::Version
64
+ version: 1.3.1
65
+ type: :runtime
66
+ prerelease: false
67
+ version_requirements: !ruby/object:Gem::Requirement
68
+ requirements:
69
+ - - "~>"
70
+ - !ruby/object:Gem::Version
71
+ version: '1.3'
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: 1.3.1
75
+ - !ruby/object:Gem::Dependency
76
+ name: hashie
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '2.0'
82
+ type: :runtime
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '2.0'
89
+ description: Scrapper for Letterboxd user's activity using public user RSS
90
+ email:
91
+ - victor.martin84@gmail.com
92
+ executables: []
93
+ extensions: []
94
+ extra_rdoc_files: []
95
+ files:
96
+ - ".gitignore"
97
+ - Gemfile
98
+ - LICENSE.txt
99
+ - README.md
100
+ - Rakefile
101
+ - lib/scrapperd.rb
102
+ - lib/scrapperd/activity.rb
103
+ - lib/scrapperd/base.rb
104
+ - lib/scrapperd/error.rb
105
+ - lib/scrapperd/version.rb
106
+ - scrapperd.gemspec
107
+ - test/activity_test.rb
108
+ - test/test_helper.rb
109
+ homepage: https://github.com/eltercero/scrapperd
110
+ licenses:
111
+ - MIT
112
+ metadata: {}
113
+ post_install_message:
114
+ rdoc_options: []
115
+ require_paths:
116
+ - lib
117
+ required_ruby_version: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - ">="
120
+ - !ruby/object:Gem::Version
121
+ version: '0'
122
+ required_rubygems_version: !ruby/object:Gem::Requirement
123
+ requirements:
124
+ - - ">="
125
+ - !ruby/object:Gem::Version
126
+ version: '0'
127
+ requirements: []
128
+ rubyforge_project:
129
+ rubygems_version: 2.2.2
130
+ signing_key:
131
+ specification_version: 4
132
+ summary: Scrapper for Letterboxd user's activity
133
+ test_files:
134
+ - test/activity_test.rb
135
+ - test/test_helper.rb