purdie 0.0.13 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/lib/purdie.rb +1 -3
- data/lib/purdie/bernard.rb +2 -4
- data/lib/purdie/cli.rb +1 -0
- data/lib/purdie/exceptions.rb +1 -3
- data/lib/purdie/item.rb +2 -6
- data/lib/purdie/license_manager.rb +0 -2
- data/lib/purdie/resolver.rb +2 -2
- data/lib/purdie/{ingester.rb → service.rb} +4 -36
- data/lib/purdie/services/flickr.rb +2 -4
- data/lib/purdie/services/soundcloud.rb +2 -3
- data/lib/purdie/services/vimeo.rb +1 -3
- data/lib/purdie/services/youtube.rb +1 -2
- data/lib/purdie/source_list.rb +31 -23
- data/lib/purdie/version.rb +1 -1
- data/spec/{bernard_spec.rb → purdie/bernard_spec.rb} +6 -0
- data/spec/{config_spec.rb → purdie/config_spec.rb} +0 -0
- data/spec/{duff_keys_spec.rb → purdie/duff_keys_spec.rb} +0 -0
- data/spec/{helpers_spec.rb → purdie/helpers_spec.rb} +0 -0
- data/spec/{item_spec.rb → purdie/item_spec.rb} +0 -6
- data/spec/{license_manager_spec.rb → purdie/license_manager_spec.rb} +0 -0
- data/spec/{resolver_spec.rb → purdie/resolver_spec.rb} +0 -0
- data/spec/{services → purdie/services}/flickr_spec.rb +4 -4
- data/spec/{services → purdie/services}/soundcloud_spec.rb +0 -14
- data/spec/{services → purdie/services}/vimeo_spec.rb +3 -3
- data/spec/{services → purdie/services}/youtube_spec.rb +3 -3
- data/spec/{source_list_spec.rb → purdie/source_list_spec.rb} +0 -0
- data/spec/vcr/Purdie_Services_Vimeo/has_yaml_for_a_video.yml +56 -0
- data/spec/vcr/Purdie_Services_YouTube/has_yaml_for_a_video.yml +1017 -0
- metadata +31 -29
- data/spec/purdie_spec.rb +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 47daab89a649c37ffa1b983b4fe593be8117d31f
|
4
|
+
data.tar.gz: bfeab60a60e4765da990339e500e6d206119b1e0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd5342cbf21b9af2b75b1dde23f86475d11c704426f5aed5098afceb55135b1ab0a54a01d9cffbe92529896316639371e432b794daa2ca30236d281c5c914085
|
7
|
+
data.tar.gz: 405010930c759d7da43155e0338bf5b2a33413f010f987029692192a2a7aefec4f16899dfde2e49114d11235145692950d17b533e13347521518408872ed1935
|
data/README.md
CHANGED
@@ -43,7 +43,7 @@ You need to create a *_sources* directory in your Jekyll project, containing fil
|
|
43
43
|
https://soundcloud.com/rawfunkmaharishi/hexaflexagon-1
|
44
44
|
https://soundcloud.com/rawfunkmaharishi/junalbandi-3
|
45
45
|
|
46
|
-
It also resolves sets/albums on
|
46
|
+
It also resolves sets/albums on all of the supported services, so this kind of thing will work:
|
47
47
|
|
48
48
|
https://www.flickr.com/photos/pikesley/sets/72157648589429938/
|
49
49
|
|
@@ -110,12 +110,12 @@ Tread carefully for now, because my metadata hacks aren't fully documented, and
|
|
110
110
|
|
111
111
|
There's no reason I couldn't support other services - I've already added [YouTube](https://github.com/rawfunkmaharishi/purdie/blob/master/spec/services/youtube_spec.rb) [support](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/youtube.rb) and others should be fairly simple. There's some introspection magic at the heart of all of this which means that as long as each service is represented by a class that:
|
112
112
|
|
113
|
-
* includes the `Purdie::
|
113
|
+
* includes the `Purdie::Service` module, and
|
114
114
|
* sports a `::matcher` class method which returns a string which will pick a URL out of an input file, and
|
115
115
|
* has a `#distill` method which takes a URL representing an item on the service and returns a hash of metadata, see e.g.
|
116
|
-
* [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb
|
117
|
-
* [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb
|
118
|
-
* [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb
|
116
|
+
* [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb)
|
117
|
+
* [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb)
|
118
|
+
* [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb)
|
119
119
|
* and optionally a `::resolve` class method which takes a set or album URL for the service and returns a list of URLs for individual items
|
120
120
|
|
121
121
|
then this should all Just Work. There's definitely a blog post in this, because Ruby introspection and metaprogramming is just mind-bogglingly powerful (and dangerous).
|
data/lib/purdie.rb
CHANGED
@@ -11,15 +11,13 @@ require 'purdie/helpers'
|
|
11
11
|
require 'purdie/bernard'
|
12
12
|
require 'purdie/config'
|
13
13
|
require 'purdie/item'
|
14
|
-
require 'purdie/ingester'
|
15
14
|
require 'purdie/source_list'
|
16
15
|
require 'purdie/license_manager'
|
17
16
|
require 'purdie/exceptions'
|
18
17
|
require 'purdie/resolver'
|
19
18
|
|
19
|
+
require 'purdie/service'
|
20
20
|
require 'purdie/services/soundcloud'
|
21
21
|
require 'purdie/services/flickr'
|
22
22
|
require 'purdie/services/vimeo'
|
23
23
|
require 'purdie/services/youtube'
|
24
|
-
|
25
|
-
Dotenv.load
|
data/lib/purdie/bernard.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
|
-
require 'purdie'
|
2
|
-
|
3
1
|
module Purdie
|
4
2
|
class Bernard
|
5
|
-
|
3
|
+
attr_accessor :verbose
|
6
4
|
|
7
5
|
def initialize
|
8
6
|
@config = Config.instance.config
|
@@ -24,7 +22,7 @@ module Purdie
|
|
24
22
|
def fetch
|
25
23
|
raise PurdieException.new 'No data sources specified' unless @sources
|
26
24
|
|
27
|
-
@sources.map { |s| SourceList.from_file s }.each do |source|
|
25
|
+
@sources.map { |s| SourceList.from_file s, @verbose }.each do |source|
|
28
26
|
source.write
|
29
27
|
end
|
30
28
|
end
|
data/lib/purdie/cli.rb
CHANGED
data/lib/purdie/exceptions.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'purdie'
|
2
|
-
|
3
1
|
module Purdie
|
4
2
|
class CredentialsException < Exception
|
5
3
|
attr_reader :service, :message
|
@@ -12,7 +10,7 @@ module Purdie
|
|
12
10
|
|
13
11
|
class PurdieException < Exception
|
14
12
|
attr_reader :message
|
15
|
-
|
13
|
+
|
16
14
|
def initialize message
|
17
15
|
@message = message
|
18
16
|
end
|
data/lib/purdie/item.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Purdie
|
2
2
|
class Item
|
3
|
-
attr_reader :url, :
|
3
|
+
attr_reader :url, :datas
|
4
4
|
|
5
5
|
def initialize url
|
6
6
|
@url = url
|
@@ -20,7 +20,7 @@ module Purdie
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def service
|
23
|
-
c =
|
23
|
+
c = Service.services.select { |s| url =~ /#{s.matcher}/ }.first
|
24
24
|
c.new
|
25
25
|
end
|
26
26
|
|
@@ -28,9 +28,5 @@ module Purdie
|
|
28
28
|
Purdie.debug 'wtf'
|
29
29
|
@datas.to_yaml
|
30
30
|
end
|
31
|
-
|
32
|
-
def source_list= source_list
|
33
|
-
@parent_file = source_list.parent_file
|
34
|
-
end
|
35
31
|
end
|
36
32
|
end
|
data/lib/purdie/resolver.rb
CHANGED
@@ -3,9 +3,9 @@ module Purdie
|
|
3
3
|
def self.resolve urls
|
4
4
|
urls = [urls].flatten
|
5
5
|
resolved = []
|
6
|
-
|
6
|
+
|
7
7
|
urls.each do |url|
|
8
|
-
service_class =
|
8
|
+
service_class = Service.services.select { |service| url =~ /#{service.matcher}/ }[0]
|
9
9
|
resolved += service_class.resolve(url) if service_class
|
10
10
|
end
|
11
11
|
|
@@ -1,27 +1,22 @@
|
|
1
1
|
Dotenv.load
|
2
2
|
|
3
3
|
module Purdie
|
4
|
-
module
|
5
|
-
attr_reader :config, :subconfig, :matcher
|
6
|
-
|
7
|
-
include Enumerable
|
8
|
-
|
4
|
+
module Service
|
9
5
|
class << self
|
10
|
-
attr_reader :
|
6
|
+
attr_reader :services
|
11
7
|
end
|
12
8
|
|
13
|
-
@
|
9
|
+
@services = []
|
14
10
|
|
15
11
|
def self.included base
|
16
12
|
# Voodoo: http://stackoverflow.com/questions/10692961/inheriting-class-methods-from-mixins
|
17
13
|
base.extend ClassMethods
|
18
|
-
@
|
14
|
+
@services.push base
|
19
15
|
end
|
20
16
|
|
21
17
|
def initialize config = nil
|
22
18
|
config = Config.instance.config unless config
|
23
19
|
@config = config
|
24
|
-
@items = []
|
25
20
|
|
26
21
|
configure
|
27
22
|
end
|
@@ -37,33 +32,6 @@ module Purdie
|
|
37
32
|
end
|
38
33
|
end
|
39
34
|
|
40
|
-
def ingest url
|
41
|
-
@items.push distill url
|
42
|
-
end
|
43
|
-
|
44
|
-
def each
|
45
|
-
@items.each do |item|
|
46
|
-
yield item
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def [] key
|
51
|
-
@items[key]
|
52
|
-
end
|
53
|
-
|
54
|
-
def to_yaml
|
55
|
-
@items.to_yaml
|
56
|
-
end
|
57
|
-
|
58
|
-
def write
|
59
|
-
if self.any?
|
60
|
-
FileUtils.mkdir_p File.dirname @output_file
|
61
|
-
File.open @output_file, 'w' do |f|
|
62
|
-
f.write self.to_yaml
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
35
|
module ClassMethods
|
68
36
|
def resolve url
|
69
37
|
[url]
|
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'purdie'
|
2
1
|
require 'flickraw-cached'
|
3
2
|
|
4
3
|
FlickRaw.api_key = ENV['FLICKR_API_KEY']
|
@@ -9,7 +8,7 @@ module Purdie
|
|
9
8
|
class Flickr
|
10
9
|
attr_accessor :size
|
11
10
|
|
12
|
-
include Purdie::
|
11
|
+
include Purdie::Service
|
13
12
|
|
14
13
|
def configure
|
15
14
|
@size = 240 # pixels
|
@@ -37,8 +36,7 @@ module Purdie
|
|
37
36
|
results['title'] = photo['title']
|
38
37
|
results['title'] = @config['default_title'] if photo['title'] == ''
|
39
38
|
results['date'] = photo['dates']['taken'].split(' ')[0]
|
40
|
-
results['photo_page'] =
|
41
|
-
# results['photo_page'] = url
|
39
|
+
results['photo_page'] = url
|
42
40
|
results['photo_url'] = FlickRaw.send(Flickr.url_for_size(@size), photo)
|
43
41
|
|
44
42
|
license = licenses.select {|l| l['id'] == photo['license']}[0]
|
@@ -1,10 +1,9 @@
|
|
1
|
-
require 'purdie'
|
2
1
|
require 'soundcloud'
|
3
2
|
|
4
3
|
module Purdie
|
5
4
|
module Services
|
6
5
|
class SoundCloud
|
7
|
-
include Purdie::
|
6
|
+
include Purdie::Service
|
8
7
|
|
9
8
|
def client
|
10
9
|
@client ||= ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
|
@@ -19,7 +18,7 @@ module Purdie
|
|
19
18
|
raise CredentialsException.new self, 'duff'
|
20
19
|
end
|
21
20
|
|
22
|
-
results = {}
|
21
|
+
results = {}
|
23
22
|
results['title'] = track['title']
|
24
23
|
results['id'] = track['id']
|
25
24
|
results['location'] = track['description']
|
data/lib/purdie/source_list.rb
CHANGED
@@ -1,52 +1,60 @@
|
|
1
|
-
require 'purdie'
|
2
|
-
|
3
1
|
module Purdie
|
4
2
|
class SourceList
|
5
3
|
include Enumerable
|
6
4
|
|
7
5
|
attr_reader :items
|
8
|
-
attr_accessor :parent_file
|
6
|
+
attr_accessor :parent_file, :verbose
|
9
7
|
|
10
8
|
def initialize sources
|
11
9
|
@sources = Resolver.resolve(sources).map { |item| Item.new item }
|
10
|
+
@items = []
|
11
|
+
@bad_creds = []
|
12
|
+
@bad_licenses = {}
|
12
13
|
end
|
13
14
|
|
14
15
|
def [] key
|
15
16
|
@sources[key]
|
16
17
|
end
|
17
18
|
|
18
|
-
def each
|
19
|
+
def each
|
19
20
|
@sources.each do |source|
|
20
21
|
yield source
|
21
22
|
end
|
22
23
|
end
|
23
24
|
|
24
25
|
def process
|
25
|
-
bad_creds = []
|
26
|
-
bad_licenses = {}
|
27
|
-
@items = []
|
28
26
|
@sources.each do |source|
|
29
27
|
begin
|
30
|
-
print "Processing #{source.url}..."
|
28
|
+
print "Processing #{source.url}... " if @verbose
|
31
29
|
source.distill
|
32
|
-
puts 'done'
|
30
|
+
puts 'done' if @verbose
|
33
31
|
rescue Purdie::CredentialsException => ce
|
34
|
-
bad_creds.push Purdie.basename(ce.service)
|
35
|
-
puts 'fail'
|
32
|
+
@bad_creds.push Purdie.basename(ce.service)
|
33
|
+
puts 'fail' if @verbose
|
34
|
+
rescue Purdie::LicenseException => le
|
35
|
+
@bad_licenses[Purdie.basename le.service].push le.name rescue @bad_licenses[Purdie.basename le.service] = [le.name]
|
36
|
+
puts 'fail' if @verbose
|
36
37
|
end
|
37
38
|
@items.push source
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
bad_credentials?
|
41
|
+
bad_licenses?
|
42
|
+
end
|
43
|
+
end
|
42
44
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
45
|
+
def bad_credentials?
|
46
|
+
if @bad_creds.any?
|
47
|
+
raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{@bad_creds.uniq.join ', '}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def bad_licenses?
|
52
|
+
if @bad_licenses.any?
|
53
|
+
bad = @bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
|
54
|
+
message = "Unknown licenses: #{bad}"
|
55
|
+
message += "\n"
|
56
|
+
message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
|
57
|
+
raise Purdie::PurdieException.new message
|
50
58
|
end
|
51
59
|
end
|
52
60
|
|
@@ -68,10 +76,10 @@ module Purdie
|
|
68
76
|
File.join ['_data', base.join('.')]
|
69
77
|
end
|
70
78
|
|
71
|
-
def self.from_file source_file
|
79
|
+
def self.from_file source_file, verbose = nil
|
72
80
|
sl = SourceList.new File.readlines(source_file).map { |l| l.strip }.select { |i| i !~ /^#/ }
|
73
81
|
sl.parent_file = source_file
|
74
|
-
|
82
|
+
sl.verbose = verbose
|
75
83
|
sl
|
76
84
|
end
|
77
85
|
end
|
data/lib/purdie/version.rb
CHANGED
@@ -2,6 +2,12 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
module Purdie
|
4
4
|
describe Bernard do
|
5
|
+
it 'does not throw a fit when initialized with no _sources dir' do
|
6
|
+
FileUtils.rmdir File.join(File.dirname(__FILE__), '..', '..', '_sources')
|
7
|
+
b = Bernard.new
|
8
|
+
expect {b.fetch}.to raise_error
|
9
|
+
end
|
10
|
+
|
5
11
|
it 'processes files', :vcr do
|
6
12
|
FileUtils.cp 'spec/support/fixtures/soundcloud.sounds', '_sources/'
|
7
13
|
FileUtils.cp 'spec/support/fixtures/vimeo.vids', '_sources/'
|
File without changes
|
File without changes
|
File without changes
|
@@ -27,12 +27,6 @@ bass: Rainey
|
|
27
27
|
)
|
28
28
|
end
|
29
29
|
|
30
|
-
it 'knows what file it came from' do
|
31
|
-
sl = instance_double('SourceList', parent_file: 'some/path/bernard.purdie')
|
32
|
-
i.source_list = sl
|
33
|
-
expect(i.parent_file).to eq 'some/path/bernard.purdie'
|
34
|
-
end
|
35
|
-
|
36
30
|
it 'knows what service it belongs to' do
|
37
31
|
expect(i.service.class).to eq Purdie::Services::SoundCloud
|
38
32
|
end
|
File without changes
|
File without changes
|
@@ -35,8 +35,8 @@ photographer: kim
|
|
35
35
|
end
|
36
36
|
|
37
37
|
it 'ingests data for a photo without a specific photographer tag', :vcr do
|
38
|
-
@f.
|
39
|
-
expect(
|
38
|
+
dist = @f.distill 'https://www.flickr.com/photos/pikesley/16649739916/'
|
39
|
+
expect(dist.to_yaml).to match /photographer: sam/
|
40
40
|
end
|
41
41
|
|
42
42
|
it 'falls back to the default photographer name', :vcr do
|
@@ -76,8 +76,8 @@ photographer: kim
|
|
76
76
|
|
77
77
|
context 'do the local config lookups' do
|
78
78
|
it 'looks up the default title', :vcr do
|
79
|
-
@f.
|
80
|
-
expect(
|
79
|
+
dist = @f.distill 'https://www.flickr.com/photos/cluttercup/16393865760/'
|
80
|
+
expect(dist['title']).to eq 'Raw Funk Maharishi'
|
81
81
|
end
|
82
82
|
end
|
83
83
|
end
|
@@ -25,20 +25,6 @@ license_url: http://creativecommons.org/licenses/by-nc-sa/4.0/
|
|
25
25
|
)
|
26
26
|
end
|
27
27
|
|
28
|
-
it 'ingests a track', :vcr do
|
29
|
-
@sc.ingest 'https://soundcloud.com/rawfunkmaharishi/bernard'
|
30
|
-
expect(@sc[0].to_yaml).to eq(
|
31
|
-
"---
|
32
|
-
title: Bernard
|
33
|
-
id: 192841052
|
34
|
-
location: Islington Academy
|
35
|
-
date: '2015-02-18'
|
36
|
-
license: Attribution-NonCommercial-ShareAlike
|
37
|
-
license_url: http://creativecommons.org/licenses/by-nc-sa/4.0/
|
38
|
-
"
|
39
|
-
)
|
40
|
-
end
|
41
|
-
|
42
28
|
context 'resolve a set' do
|
43
29
|
it 'resolves a set from a url', :vcr do
|
44
30
|
set = SoundCloud.resolve 'https://soundcloud.com/rawfunkmaharishi/sets/islington-academy-sessions'
|