purdie 0.0.13 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +5 -5
  3. data/lib/purdie.rb +1 -3
  4. data/lib/purdie/bernard.rb +2 -4
  5. data/lib/purdie/cli.rb +1 -0
  6. data/lib/purdie/exceptions.rb +1 -3
  7. data/lib/purdie/item.rb +2 -6
  8. data/lib/purdie/license_manager.rb +0 -2
  9. data/lib/purdie/resolver.rb +2 -2
  10. data/lib/purdie/{ingester.rb → service.rb} +4 -36
  11. data/lib/purdie/services/flickr.rb +2 -4
  12. data/lib/purdie/services/soundcloud.rb +2 -3
  13. data/lib/purdie/services/vimeo.rb +1 -3
  14. data/lib/purdie/services/youtube.rb +1 -2
  15. data/lib/purdie/source_list.rb +31 -23
  16. data/lib/purdie/version.rb +1 -1
  17. data/spec/{bernard_spec.rb → purdie/bernard_spec.rb} +6 -0
  18. data/spec/{config_spec.rb → purdie/config_spec.rb} +0 -0
  19. data/spec/{duff_keys_spec.rb → purdie/duff_keys_spec.rb} +0 -0
  20. data/spec/{helpers_spec.rb → purdie/helpers_spec.rb} +0 -0
  21. data/spec/{item_spec.rb → purdie/item_spec.rb} +0 -6
  22. data/spec/{license_manager_spec.rb → purdie/license_manager_spec.rb} +0 -0
  23. data/spec/{resolver_spec.rb → purdie/resolver_spec.rb} +0 -0
  24. data/spec/{services → purdie/services}/flickr_spec.rb +4 -4
  25. data/spec/{services → purdie/services}/soundcloud_spec.rb +0 -14
  26. data/spec/{services → purdie/services}/vimeo_spec.rb +3 -3
  27. data/spec/{services → purdie/services}/youtube_spec.rb +3 -3
  28. data/spec/{source_list_spec.rb → purdie/source_list_spec.rb} +0 -0
  29. data/spec/vcr/Purdie_Services_Vimeo/has_yaml_for_a_video.yml +56 -0
  30. data/spec/vcr/Purdie_Services_YouTube/has_yaml_for_a_video.yml +1017 -0
  31. metadata +31 -29
  32. data/spec/purdie_spec.rb +0 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f65941fa63765aacc68d10628f57455119d907ff
4
- data.tar.gz: 663003c250b66bed25aad0967c9d770bd6c7890b
3
+ metadata.gz: 47daab89a649c37ffa1b983b4fe593be8117d31f
4
+ data.tar.gz: bfeab60a60e4765da990339e500e6d206119b1e0
5
5
  SHA512:
6
- metadata.gz: f0ed75ddf476ad8c33942148d48b11e1ebde71e827d45c36987bcc51f0440b001ab603278b45b4091613ae4560904a06156463a2e50353ea3172b03b6de26de6
7
- data.tar.gz: e123f933c514079cb7cd5ddcb23d987f9605e97bbecd5ca4f4e477ef628e2e89b4300b768be378a79ca0a64b199973d4b1118122b825646bbba40a1383c182e7
6
+ metadata.gz: fd5342cbf21b9af2b75b1dde23f86475d11c704426f5aed5098afceb55135b1ab0a54a01d9cffbe92529896316639371e432b794daa2ca30236d281c5c914085
7
+ data.tar.gz: 405010930c759d7da43155e0338bf5b2a33413f010f987029692192a2a7aefec4f16899dfde2e49114d11235145692950d17b533e13347521518408872ed1935
data/README.md CHANGED
@@ -43,7 +43,7 @@ You need to create a *_sources* directory in your Jekyll project, containing fil
43
43
  https://soundcloud.com/rawfunkmaharishi/hexaflexagon-1
44
44
  https://soundcloud.com/rawfunkmaharishi/junalbandi-3
45
45
 
46
- It also resolves sets/albums on Flickr and SoundCloud (and soon Vimeo), so this kind of thing will work:
46
+ It also resolves sets/albums on all of the supported services, so this kind of thing will work:
47
47
 
48
48
  https://www.flickr.com/photos/pikesley/sets/72157648589429938/
49
49
 
@@ -110,12 +110,12 @@ Tread carefully for now, because my metadata hacks aren't fully documented, and
110
110
 
111
111
  There's no reason I couldn't support other services - I've already added [YouTube](https://github.com/rawfunkmaharishi/purdie/blob/master/spec/services/youtube_spec.rb) [support](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/youtube.rb) and others should be fairly simple. There's some introspection magic at the heart of all of this which means that as long as each service is represented by a class that:
112
112
 
113
- * includes the `Purdie::Ingester` module, and
113
+ * includes the `Purdie::Service` module, and
114
114
  * sports a `::matcher` class method which returns a string which will pick a URL out of an input file, and
115
115
  * has a `#distill` method which takes a URL representing an item on the service and returns a hash of metadata, see e.g.
116
- * [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb#L33)
117
- * [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb#L13)
118
- * [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb#L20)
116
+ * [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb)
117
+ * [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb)
118
+ * [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb)
119
119
  * and optionally a `::resolve` class method which takes a set or album URL for the service and returns a list of URLs for individual items
120
120
 
121
121
  then this should all Just Work. There's definitely a blog post in this, because Ruby introspection and metaprogramming is just mind-bogglingly powerful (and dangerous).
@@ -11,15 +11,13 @@ require 'purdie/helpers'
11
11
  require 'purdie/bernard'
12
12
  require 'purdie/config'
13
13
  require 'purdie/item'
14
- require 'purdie/ingester'
15
14
  require 'purdie/source_list'
16
15
  require 'purdie/license_manager'
17
16
  require 'purdie/exceptions'
18
17
  require 'purdie/resolver'
19
18
 
19
+ require 'purdie/service'
20
20
  require 'purdie/services/soundcloud'
21
21
  require 'purdie/services/flickr'
22
22
  require 'purdie/services/vimeo'
23
23
  require 'purdie/services/youtube'
24
-
25
- Dotenv.load
@@ -1,8 +1,6 @@
1
- require 'purdie'
2
-
3
1
  module Purdie
4
2
  class Bernard
5
- attr_reader :config
3
+ attr_accessor :verbose
6
4
 
7
5
  def initialize
8
6
  @config = Config.instance.config
@@ -24,7 +22,7 @@ module Purdie
24
22
  def fetch
25
23
  raise PurdieException.new 'No data sources specified' unless @sources
26
24
 
27
- @sources.map { |s| SourceList.from_file s }.each do |source|
25
+ @sources.map { |s| SourceList.from_file s, @verbose }.each do |source|
28
26
  source.write
29
27
  end
30
28
  end
@@ -14,6 +14,7 @@ module Purdie
14
14
  :desc => 'Specify source file to process'
15
15
  def fetch
16
16
  b = Bernard.new
17
+ b.verbose = true
17
18
  # b.source_file = options[:source_file] if options[:source_file]
18
19
 
19
20
  begin
@@ -1,5 +1,3 @@
1
- require 'purdie'
2
-
3
1
  module Purdie
4
2
  class CredentialsException < Exception
5
3
  attr_reader :service, :message
@@ -12,7 +10,7 @@ module Purdie
12
10
 
13
11
  class PurdieException < Exception
14
12
  attr_reader :message
15
-
13
+
16
14
  def initialize message
17
15
  @message = message
18
16
  end
@@ -1,6 +1,6 @@
1
1
  module Purdie
2
2
  class Item
3
- attr_reader :url, :parent_file, :datas
3
+ attr_reader :url, :datas
4
4
 
5
5
  def initialize url
6
6
  @url = url
@@ -20,7 +20,7 @@ module Purdie
20
20
  end
21
21
 
22
22
  def service
23
- c = Ingester.ingesters.select { |s| url =~ /#{s.matcher}/ }.first
23
+ c = Service.services.select { |s| url =~ /#{s.matcher}/ }.first
24
24
  c.new
25
25
  end
26
26
 
@@ -28,9 +28,5 @@ module Purdie
28
28
  Purdie.debug 'wtf'
29
29
  @datas.to_yaml
30
30
  end
31
-
32
- def source_list= source_list
33
- @parent_file = source_list.parent_file
34
- end
35
31
  end
36
32
  end
@@ -1,5 +1,3 @@
1
- require 'purdie'
2
-
3
1
  module Purdie
4
2
  class LicenseManager
5
3
  LOOKUPS = YAML.load File.read File.join(File.dirname(__FILE__), '..', '..', '_config/licenses.yaml')
@@ -3,9 +3,9 @@ module Purdie
3
3
  def self.resolve urls
4
4
  urls = [urls].flatten
5
5
  resolved = []
6
-
6
+
7
7
  urls.each do |url|
8
- service_class = Ingester.ingesters.select { |service| url =~ /#{service.matcher}/ }[0]
8
+ service_class = Service.services.select { |service| url =~ /#{service.matcher}/ }[0]
9
9
  resolved += service_class.resolve(url) if service_class
10
10
  end
11
11
 
@@ -1,27 +1,22 @@
1
1
  Dotenv.load
2
2
 
3
3
  module Purdie
4
- module Ingester
5
- attr_reader :config, :subconfig, :matcher
6
-
7
- include Enumerable
8
-
4
+ module Service
9
5
  class << self
10
- attr_reader :ingesters
6
+ attr_reader :services
11
7
  end
12
8
 
13
- @ingesters = []
9
+ @services = []
14
10
 
15
11
  def self.included base
16
12
  # Voodoo: http://stackoverflow.com/questions/10692961/inheriting-class-methods-from-mixins
17
13
  base.extend ClassMethods
18
- @ingesters.push base
14
+ @services.push base
19
15
  end
20
16
 
21
17
  def initialize config = nil
22
18
  config = Config.instance.config unless config
23
19
  @config = config
24
- @items = []
25
20
 
26
21
  configure
27
22
  end
@@ -37,33 +32,6 @@ module Purdie
37
32
  end
38
33
  end
39
34
 
40
- def ingest url
41
- @items.push distill url
42
- end
43
-
44
- def each
45
- @items.each do |item|
46
- yield item
47
- end
48
- end
49
-
50
- def [] key
51
- @items[key]
52
- end
53
-
54
- def to_yaml
55
- @items.to_yaml
56
- end
57
-
58
- def write
59
- if self.any?
60
- FileUtils.mkdir_p File.dirname @output_file
61
- File.open @output_file, 'w' do |f|
62
- f.write self.to_yaml
63
- end
64
- end
65
- end
66
-
67
35
  module ClassMethods
68
36
  def resolve url
69
37
  [url]
@@ -1,4 +1,3 @@
1
- require 'purdie'
2
1
  require 'flickraw-cached'
3
2
 
4
3
  FlickRaw.api_key = ENV['FLICKR_API_KEY']
@@ -9,7 +8,7 @@ module Purdie
9
8
  class Flickr
10
9
  attr_accessor :size
11
10
 
12
- include Purdie::Ingester
11
+ include Purdie::Service
13
12
 
14
13
  def configure
15
14
  @size = 240 # pixels
@@ -37,8 +36,7 @@ module Purdie
37
36
  results['title'] = photo['title']
38
37
  results['title'] = @config['default_title'] if photo['title'] == ''
39
38
  results['date'] = photo['dates']['taken'].split(' ')[0]
40
- results['photo_page'] = photo['urls'][0]['_content']
41
- # results['photo_page'] = url
39
+ results['photo_page'] = url
42
40
  results['photo_url'] = FlickRaw.send(Flickr.url_for_size(@size), photo)
43
41
 
44
42
  license = licenses.select {|l| l['id'] == photo['license']}[0]
@@ -1,10 +1,9 @@
1
- require 'purdie'
2
1
  require 'soundcloud'
3
2
 
4
3
  module Purdie
5
4
  module Services
6
5
  class SoundCloud
7
- include Purdie::Ingester
6
+ include Purdie::Service
8
7
 
9
8
  def client
10
9
  @client ||= ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
@@ -19,7 +18,7 @@ module Purdie
19
18
  raise CredentialsException.new self, 'duff'
20
19
  end
21
20
 
22
- results = {}
21
+ results = {}
23
22
  results['title'] = track['title']
24
23
  results['id'] = track['id']
25
24
  results['location'] = track['description']
@@ -1,9 +1,7 @@
1
- require 'purdie'
2
-
3
1
  module Purdie
4
2
  module Services
5
3
  class Vimeo
6
- include Purdie::Ingester
4
+ include Purdie::Service
7
5
 
8
6
  def get url
9
7
  @id = Purdie.get_id url
@@ -1,10 +1,9 @@
1
- require 'purdie'
2
1
  require 'google/api_client'
3
2
 
4
3
  module Purdie
5
4
  module Services
6
5
  class YouTube
7
- include Ingester
6
+ include Service
8
7
 
9
8
  API_SERVICE_NAME = 'youtube'
10
9
  API_VERSION = 'v3'
@@ -1,52 +1,60 @@
1
- require 'purdie'
2
-
3
1
  module Purdie
4
2
  class SourceList
5
3
  include Enumerable
6
4
 
7
5
  attr_reader :items
8
- attr_accessor :parent_file
6
+ attr_accessor :parent_file, :verbose
9
7
 
10
8
  def initialize sources
11
9
  @sources = Resolver.resolve(sources).map { |item| Item.new item }
10
+ @items = []
11
+ @bad_creds = []
12
+ @bad_licenses = {}
12
13
  end
13
14
 
14
15
  def [] key
15
16
  @sources[key]
16
17
  end
17
18
 
18
- def each &block
19
+ def each
19
20
  @sources.each do |source|
20
21
  yield source
21
22
  end
22
23
  end
23
24
 
24
25
  def process
25
- bad_creds = []
26
- bad_licenses = {}
27
- @items = []
28
26
  @sources.each do |source|
29
27
  begin
30
- print "Processing #{source.url}..."
28
+ print "Processing #{source.url}... " if @verbose
31
29
  source.distill
32
- puts 'done'
30
+ puts 'done' if @verbose
33
31
  rescue Purdie::CredentialsException => ce
34
- bad_creds.push Purdie.basename(ce.service)
35
- puts 'fail'
32
+ @bad_creds.push Purdie.basename(ce.service)
33
+ puts 'fail' if @verbose
34
+ rescue Purdie::LicenseException => le
35
+ @bad_licenses[Purdie.basename le.service].push le.name rescue @bad_licenses[Purdie.basename le.service] = [le.name]
36
+ puts 'fail' if @verbose
36
37
  end
37
38
  @items.push source
38
39
 
39
- if bad_creds.any?
40
- raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{bad_creds.uniq.join ', '}"
41
- end
40
+ bad_credentials?
41
+ bad_licenses?
42
+ end
43
+ end
42
44
 
43
- if bad_licenses.any?
44
- bad = bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
45
- message = "Unknown licenses: #{bad}"
46
- message += "\n"
47
- message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
48
- raise Purdie::PurdieException.new message
49
- end
45
+ def bad_credentials?
46
+ if @bad_creds.any?
47
+ raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{@bad_creds.uniq.join ', '}"
48
+ end
49
+ end
50
+
51
+ def bad_licenses?
52
+ if @bad_licenses.any?
53
+ bad = @bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
54
+ message = "Unknown licenses: #{bad}"
55
+ message += "\n"
56
+ message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
57
+ raise Purdie::PurdieException.new message
50
58
  end
51
59
  end
52
60
 
@@ -68,10 +76,10 @@ module Purdie
68
76
  File.join ['_data', base.join('.')]
69
77
  end
70
78
 
71
- def self.from_file source_file
79
+ def self.from_file source_file, verbose = nil
72
80
  sl = SourceList.new File.readlines(source_file).map { |l| l.strip }.select { |i| i !~ /^#/ }
73
81
  sl.parent_file = source_file
74
-
82
+ sl.verbose = verbose
75
83
  sl
76
84
  end
77
85
  end
@@ -1,3 +1,3 @@
1
1
  module Purdie
2
- VERSION = "0.0.13"
2
+ VERSION = "0.1.0"
3
3
  end
@@ -2,6 +2,12 @@ require 'spec_helper'
2
2
 
3
3
  module Purdie
4
4
  describe Bernard do
5
+ it 'does not throw a fit when initialized with no _sources dir' do
6
+ FileUtils.rmdir File.join(File.dirname(__FILE__), '..', '..', '_sources')
7
+ b = Bernard.new
8
+ expect {b.fetch}.to raise_error
9
+ end
10
+
5
11
  it 'processes files', :vcr do
6
12
  FileUtils.cp 'spec/support/fixtures/soundcloud.sounds', '_sources/'
7
13
  FileUtils.cp 'spec/support/fixtures/vimeo.vids', '_sources/'
@@ -27,12 +27,6 @@ bass: Rainey
27
27
  )
28
28
  end
29
29
 
30
- it 'knows what file it came from' do
31
- sl = instance_double('SourceList', parent_file: 'some/path/bernard.purdie')
32
- i.source_list = sl
33
- expect(i.parent_file).to eq 'some/path/bernard.purdie'
34
- end
35
-
36
30
  it 'knows what service it belongs to' do
37
31
  expect(i.service.class).to eq Purdie::Services::SoundCloud
38
32
  end
@@ -35,8 +35,8 @@ photographer: kim
35
35
  end
36
36
 
37
37
  it 'ingests data for a photo without a specific photographer tag', :vcr do
38
- @f.ingest 'https://www.flickr.com/photos/pikesley/16649739916/'
39
- expect(@f[0].to_yaml).to match /photographer: sam/
38
+ dist = @f.distill 'https://www.flickr.com/photos/pikesley/16649739916/'
39
+ expect(dist.to_yaml).to match /photographer: sam/
40
40
  end
41
41
 
42
42
  it 'falls back to the default photographer name', :vcr do
@@ -76,8 +76,8 @@ photographer: kim
76
76
 
77
77
  context 'do the local config lookups' do
78
78
  it 'looks up the default title', :vcr do
79
- @f.ingest 'https://www.flickr.com/photos/cluttercup/16393865760/'
80
- expect(@f[0]['title']).to eq 'Raw Funk Maharishi'
79
+ dist = @f.distill 'https://www.flickr.com/photos/cluttercup/16393865760/'
80
+ expect(dist['title']).to eq 'Raw Funk Maharishi'
81
81
  end
82
82
  end
83
83
  end
@@ -25,20 +25,6 @@ license_url: http://creativecommons.org/licenses/by-nc-sa/4.0/
25
25
  )
26
26
  end
27
27
 
28
- it 'ingests a track', :vcr do
29
- @sc.ingest 'https://soundcloud.com/rawfunkmaharishi/bernard'
30
- expect(@sc[0].to_yaml).to eq(
31
- "---
32
- title: Bernard
33
- id: 192841052
34
- location: Islington Academy
35
- date: '2015-02-18'
36
- license: Attribution-NonCommercial-ShareAlike
37
- license_url: http://creativecommons.org/licenses/by-nc-sa/4.0/
38
- "
39
- )
40
- end
41
-
42
28
  context 'resolve a set' do
43
29
  it 'resolves a set from a url', :vcr do
44
30
  set = SoundCloud.resolve 'https://soundcloud.com/rawfunkmaharishi/sets/islington-academy-sessions'