purdie 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4e6e972e93cf8cdd2ec9d9eeef1b617445d7a20
-  data.tar.gz: 44a7bb6b0d4030f7a27a03fe85f827df4b1d4942
+  metadata.gz: c7ede05eac12185b11e1038f17880897a4341680
+  data.tar.gz: 51528843daf81c4e7e17e503861a971dd224ab9e
 SHA512:
-  metadata.gz: bbc8b0189d139240c47f2916cbdced45818c7b7c9480e2c4e22bbf45876953eea012bf040b06276585c4a6b507c6437dc08560b49c789e281aaa0efee9b5bf00
-  data.tar.gz: 096c577dad0b09883efbfded82434faac5f6034066e98e1a221d6b94d10fa2bd229f1764ee89be8e8709f360d0a61107f6c63a4bd1dc1eb0c4510de84069363b
+  metadata.gz: bd59f0cb292e5626b3f852cb47285b75f2864afc23417797d5ce1d46ede91b83a99ebf8162667e1e059d35ce0bea40b5acbe5201f09d2ac149cd994ceab1e53c
+  data.tar.gz: 0d6759f0aa70f5a59a54645628b549c2d4951fdc204a56e76f343af7c0e57610ce7d8cd23872e67d5b7270647a310282c72b200447083e9d61a78b900c58bcaf

data/.gitignore

@@ -13,4 +13,5 @@
 *.a
 mkmf.log
 .env
-cruft.rb
+_data/
+*log

data/.travis.yml

@@ -11,3 +11,4 @@ env:
   - secure: AeXUYDzqttkNCkemmsAn3NkYUMwcLX7h5QNvYQ5UNrPkDTEkW/TGrgAfgErSmMRCiQas+jqRm4CJOGA917gPZrkbdxHf4MwF7vvACQPMmwFLhW7ZVERrRjV/jJNMVsMKb4Jr1SuhfCgrNWKOvowKGRM0ulqbkda/sgM0QtH/Iow=
   - secure: jIVGrSlocGRVRXfiI1tnsa6bMWkQA2Cev5R7+W5nIiPmO9piH6YBFKxvbpeebvSDpvB7wVzrAM3JUeCaYXXF7hiDT7NAYiU8+v6FB20bvyFM9cDxtYB+SJRTwsF2Mq8212bMg47hdubnSFL0g9IC/nLtxq9tJjGycbMaAMZRKC4=
   - secure: aKZ3c8pttF/tDAbQ+zfrflDRPF7I6lTLV8rHm4NwNh1cEgOv8v45HiBWIl5IplaNXonUcfhrvrmi11m9KhWtKiCiA7Z1BHJJu9eP+QhedJtGvxbmnDQ1UOXAY2Ot9Y1b0wRa2YqsYIZOUg42erVsHcWiPXWpfCQ+27dqglP59k8=
+  - secure: R4uwxMF7CtpRtDZOcan2caJci0p6Aum0FOdu72kut+omnFjumIhtmVVh2BvUZW5pD9LoqCreYHVifSx8nsbRvmMbZ67I3Y0JFst+QfHDCw8vQYBJ3oAUkWZf8wo4zW3GaAHmfqOrhIPifSa/rMNgvK95g/Dd9Fnr/Dyto2bRVG0=

data/README.md

@@ -42,8 +42,17 @@ You need to create a *_sources* directory in your Jekyll project, containing fil
 
     https://soundcloud.com/rawfunkmaharishi/hexaflexagon-1
     https://soundcloud.com/rawfunkmaharishi/junalbandi-3
+    
+It also resolves sets/albums on Flickr and SoundCloud (and soon Vimeo), so this will work:
 
-(you can alternatively specify a source file on the command line with the `-f` flag).
+    https://www.flickr.com/photos/pikesley/sets/72157648589429938/
+    https://soundcloud.com/rawfunkmaharishi/sets/islington-academy-sessions
+
+####Notes about *_sources*:
+    
+* Purdie does not care how many files are in *_sources*, nor if the services are all mixed up together in those files - it will simply dump out one file per service
+* If a URL appears multiple times in a resolved list, only the first appearance will be propagated to the output
+* You can specify a source file on the command line with the `-f` flag
 
 You also need a *.env* file with the relevant credentials in it:
 
@@ -51,17 +60,25 @@ You also need a *.env* file with the relevant credentials in it:
     FLICKR_SECRET: this_a_secret
 
     SOUNDCLOUD_CLIENT_ID: this_a_client_id
-    SOUNDCLOUD_USER_ID: this_a_user_id
 
     VIMEO_BEARER_TOKEN: this_is_bearer_token
+    
+    YOUTUBE_API_KEY: this_is_key_for_youtube
 
-(get those things from [Flickr](https://www.flickr.com/services/apps/create/apply), [SoundCloud](http://soundcloud.com/you/apps/new) and [Vimeo](https://developer.vimeo.com/apps/new))
+(get those things from [Flickr](https://www.flickr.com/services/apps/create/apply), [SoundCloud](http://soundcloud.com/you/apps/new), [Vimeo](https://developer.vimeo.com/apps/new) and [YouTube](https://console.developers.google.com/project))
 
 And then you can run
 
     purdie fetch
 
-and it will dump out YAML files into *_data*, ready for Jekyll to consume.
+(`fetch` is the default task (in fact currently the only task), so just `purdie` will work) and it will dump out YAML files into *_data*:
+    
+    flickr.yaml
+    soundcloud.yaml
+    vimeo.yaml
+    youtube.yaml
+    
+ready for Jekyll to consume.
 
 ###Customisation
 
@@ -87,21 +104,18 @@ Tread carefully for now, because my metadata hacks aren't fully documented, and
 
 ##What next?
 
-There's no reason I couldn't support other services - I've now added [YouTube](https://github.com/rawfunkmaharishi/purdie/blob/master/spec/services/youtube_spec.rb) [support](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/youtube.rb) and others should be fairly simple. There's some introspection magic at the heart of all of this which means that as long as each service is represented by a class that:
+There's no reason I couldn't support other services - I've already added [YouTube](https://github.com/rawfunkmaharishi/purdie/blob/master/spec/services/youtube_spec.rb) [support](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/youtube.rb) and others should be fairly simple. There's some introspection magic at the heart of all of this which means that as long as each service is represented by a class that:
 
 * includes the `Purdie::Ingester` module, and
 * sports a `::matcher` class method which returns a string which will pick a URL out of an input file, and
 * has a `#distill` method which takes a URL representing an item on the service and returns a hash of metadata, see e.g.
   * [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb#L27)
-  * [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb#L31)
-  * [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb#L28)
+  * [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb#L13)
+  * [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb#L15)
+* and optionally a `::resolve` class method which takes a set or album URL for the service and returns a list of URLs for individual items
 
 then this should all Just Work. There's definitely a blog post in this, because Ruby introspection and metaprogramming is just mind-bogglingly powerful (and dangerous).
 
-More prosaically, I think it will be fairly simple to support Flickr sets and SoundCloud albums as input data.
-
-And I can definitely rationalise these [horrible license lookups](https://github.com/rawfunkmaharishi/purdie/blob/master/_config/defaults.yaml#L5) into a module or even a gem of their own.
-
 And of course, known issues are [here](https://github.com/rawfunkmaharishi/purdie/issues).
 
 ##Why Purdie?

data/_config/defaults.yaml

@@ -1,24 +1,3 @@
 source_dir:    _sources
 output_dir:    _data
 default_title: Bernard Purdie, the World's most-recorded drummer
-
-# those unwieldy lookups are because each service calls the CC licenses by a slightly different name and defaults to a different version of the license. I think I need to abstract all of this out into a modlue or even a gem of its own
-license_lookups:
-
-  # SoundCloud
-  cc-by-nc-sa:
-    short_name: BY-NC-SA
-    full_name:  Attribution-NonCommercial-ShareAlike
-    url:        http://creativecommons.org/licenses/by-nc-sa/4.0/
-
-  # Vimeo
-  by-nc-sa:
-    short_name: BY-NC-SA
-    full_name:  Attribution-NonCommercial-ShareAlike
-    url:        http://creativecommons.org/licenses/by-nc-sa/3.0/
-
-  # This one is YouTube. WTF YouTube?
-  creativeCommon:
-    short_name: BY
-    full_name:  Attribution
-    url:        https://creativecommons.org/licenses/by/3.0/

data/_config/licenses.yaml

@@ -0,0 +1,33 @@
+Flickr:
+  '1':
+    short_name: BY-NC-SA
+    full_name:  Attribution-NonCommercial-ShareAlike
+    url:        https://creativecommons.org/licenses/by-nc-sa/2.0/
+
+  '2':
+    short_name: BY-NC
+    full_name:  Attribution-NonCommercial
+    url:        http://creativecommons.org/licenses/by-nc/2.0/
+
+SoundCloud:
+  cc-by-nc-sa:
+    short_name: BY-NC-SA
+    full_name:  Attribution-NonCommercial-ShareAlike
+    url:        http://creativecommons.org/licenses/by-nc-sa/4.0/
+
+Vimeo:
+  by-nc-sa:
+    short_name: BY-NC-SA
+    full_name:  Attribution-NonCommercial-ShareAlike
+    url:        http://creativecommons.org/licenses/by-nc-sa/3.0/
+
+YouTube:
+  creativeCommon:
+    short_name: BY
+    full_name:  Attribution
+    url:        https://creativecommons.org/licenses/by/3.0/
+
+  youtube:
+    short_name: YouTube
+    full_name:  YouTube
+    url:        https://www.youtube.com/static?gl=GB&template=terms

data/features/duff-credentials.feature

@@ -0,0 +1,33 @@
+@vcr
+
+Feature: Missing or bad credentials
+
+  Scenario: Attempt to process SoundCloud URLs with missing credentials
+    Given a file named "_sources/soundcloud.csv" with:
+    """
+    https://soundcloud.com/rawfunkmaharishi/hexaflexagon-1
+    """
+    And no credentials for SoundCloud
+    When I run `purdie fetch`
+    Then the stderr should contain:
+    """
+    Missing or duff credentials for: SoundCloud
+    """
+    And I restore the credentials for SoundCloud
+
+  Scenario: Bad credentials for multiple services
+    Given a file named "_sources/sources.sources" with:
+    """
+    https://vimeo.com/110132671
+    https://vimeo.com/110133117
+    https://www.youtube.com/watch?v=Qt_J0jNqtZg&list=PLuPLM2FI60-M0-aWejF9WgB-Dkt1TuQXv&index=2
+    """
+    And no credentials for Vimeo
+    And bad credentials for YouTube
+    When I run `purdie fetch`
+    Then the stderr should contain:
+    """
+    Missing or duff credentials for: Vimeo, YouTube
+    """
+    And I restore the credentials for Vimeo
+    And I restore the credentials for YouTube

data/features/flickr-album.feature

@@ -17,7 +17,7 @@ Feature: Flickr album
       photo_url: https://farm8.staticflickr.com/7506/16252009191_ea2d06e6bb_m.jpg
       license: Attribution-NonCommercial-ShareAlike
       license_url: https://creativecommons.org/licenses/by-nc-sa/2.0/
-      photographer: pikesley
+      photographer: sam
     """
     And a file named "_data/soundcloud.yaml" should not exist
     And a file named "_data/vimeo.yaml" should not exist

data/features/flickr-with-lookups.feature

@@ -0,0 +1,29 @@
+@vcr
+Feature: Flickr
+
+  Scenario: Generate Flickr YAML with lookups
+    Given a file named "_sources/flickr.csv" with:
+    """
+    https://www.flickr.com/photos/cluttercup/16393865760/
+    """
+    And a file named "_config/purdie.yaml" with:
+    """
+    photographer_lookups:
+      cluttercup: jane
+
+    default_title: Raw Funk Maharishi
+    """
+    When I successfully run `purdie fetch`
+    Then a file named "_data/flickr.yaml" should exist
+    And the file "_data/flickr.yaml" should contain:
+    """
+    - title: Raw Funk Maharishi
+      date: '2015-02-18'
+      photo_page: https://www.flickr.com/photos/cluttercup/16393865760/
+      photo_url: https://farm8.staticflickr.com/7395/16393865760_bfbfc1c267_m.jpg
+      license: Attribution-NonCommercial
+      license_url: https://creativecommons.org/licenses/by-nc/2.0/
+      photographer: jane
+    """
+    And a file named "_data/soundcloud.yaml" should not exist
+    And a file named "_data/vimeo.yaml" should not exist

data/features/rfm.feature

@@ -28,14 +28,14 @@ Feature: First contact with the Real World
       photo_url: https://farm8.staticflickr.com/7398/15950875724_23d58be214_m.jpg
       license: Attribution-NonCommercial
       license_url: https://creativecommons.org/licenses/by-nc/2.0/
-      photographer: jane
+      photographer: cluttercup
     - title: Raw Funk Maharishi
       date: '2015-02-18'
       photo_page: https://www.flickr.com/photos/cluttercup/16579675721/
       photo_url: https://farm8.staticflickr.com/7418/16579675721_f765c42f99_m.jpg
       license: Attribution-NonCommercial
       license_url: https://creativecommons.org/licenses/by-nc/2.0/
-      photographer: jane
+      photographer: cluttercup
     """
     And a file named "_data/soundcloud.yaml" should exist
     And the file "_data/soundcloud.yaml" should contain:

data/features/step_definitions/purdie_steps.rb

@@ -0,0 +1,26 @@
+Given /^no credentials for SoundCloud$/ do
+  $sc = ENV['SOUNDCLOUD_CLIENT_ID']
+  ENV['SOUNDCLOUD_CLIENT_ID'] = nil
+end
+
+Given /^I restore the credentials for SoundCloud$/ do
+  ENV['SOUNDCLOUD_CLIENT_ID'] = $sc
+end
+
+Given /^no credentials for Vimeo$/ do
+  $v = ENV['VIMEO_BEARER_TOKEN']
+  ENV['VIMEO_BEARER_TOKEN'] = nil
+end
+
+Given /^bad credentials for YouTube$/  do
+  $y = ENV['YOUTUBE_API_KEY']
+  ENV['YOUTUBE_API_KEY'] = 'fji423cj234'
+end
+
+Given /^I restore the credentials for Vimeo$/  do
+  ENV['VIMEO_BEARER_TOKEN'] = $v
+end
+
+Given /^I restore the credentials for YouTube$/  do
+  ENV['YOUTUBE_API_KEY'] = $y
+end

data/features/support/fixtures/_config/purdie.yaml

@@ -1,4 +1,4 @@
 default_title: Raw Funk Maharishi
 
 photographer_lookups:
-  cluttercup: jane
+  pikesley: sam

data/features/support/fixtures/vcr/A_URL_we_don_t_understand/Handle_a_URL_we_don_t_recognise.yml

@@ -15,8 +15,8 @@ http_interactions:
       - FlickRaw/0.9.8
       Authorization:
       - OAuth realm="https://api.flickr.com/services/rest/", oauth_consumer_key="<FLICKR_API_KEY>",
-        oauth_nonce="BhHMbdRV8L2SQzpYSRL%2FWCxnjGwaREmYEq3r26OG2rE%3D", oauth_signature="<FLICKR_SECRET>%26",
-        oauth_signature_method="PLAINTEXT", oauth_timestamp="1426006401", oauth_token="",
+        oauth_nonce="lStJS%2BN%2FCaUkNeTkqy96sNrrbLb4jG9KHEIFoqANl2o%3D", oauth_signature="<FLICKR_SECRET>%26",
+        oauth_signature_method="PLAINTEXT", oauth_timestamp="1426093984", oauth_token="",
         oauth_version="1.0"
       Content-Type:
       - application/x-www-form-urlencoded
@@ -26,7 +26,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Tue, 10 Mar 2015 16:53:22 GMT
+      - Wed, 11 Mar 2015 17:13:05 GMT
       Content-Type:
       - application/json
       Content-Length:
@@ -38,14 +38,14 @@ http_interactions:
       Cache-Control:
       - private
       X-Served-By:
-      - www55.flickr.bf1.yahoo.com
+      - www58.flickr.bf1.yahoo.com
       Vary:
       - Accept-Encoding
       Age:
-      - '2'
+      - '0'
       Via:
-      - http/1.1 fts110.flickr.bf1.yahoo.com (ApacheTrafficServer/4.0.2 [cMsSf ]),
-        http/1.1 r12.ycpi.dea.yahoo.net (ApacheTrafficServer [cMsSf ])
+      - http/1.1 fts116.flickr.bf1.yahoo.com (ApacheTrafficServer/4.0.2 [cMsSf ]),
+        http/1.1 r21.ycpi.dea.yahoo.net (ApacheTrafficServer [cMsSf ])
       Server:
       - ATS
       Connection:
@@ -58,7 +58,7 @@ http_interactions:
         15:54:24","takengranularity":"0","takenunknown":"0","lastupdate":"1416111853"},"views":"94","editability":{"cancomment":0,"canaddmeta":0},"publiceditability":{"cancomment":1,"canaddmeta":0},"usage":{"candownload":1,"canblog":0,"canprint":0,"canshare":1},"comments":{"_content":"0"},"notes":{"note":[]},"people":{"haspeople":0},"tags":{"tag":[{"id":"128898522-15631479625-1422028","author":"128943844@N06","authorname":"rawfunkmaharishi","raw":"the
         comedy","_content":"thecomedy","machine_tag":0},{"id":"128898522-15631479625-17975","author":"128943844@N06","authorname":"rawfunkmaharishi","raw":"gigs","_content":"gigs","machine_tag":0},{"id":"128898522-15631479625-228989594","author":"128943844@N06","authorname":"rawfunkmaharishi","raw":"photographer:kim","_content":"photographerkim","machine_tag":0}]},"urls":{"url":[{"type":"photopage","_content":"https:\/\/www.flickr.com\/photos\/rawfunkmaharishi\/15631479625\/"}]},"media":"photo"},"stat":"ok"}'
     http_version: 
-  recorded_at: Tue, 10 Mar 2015 16:53:22 GMT
+  recorded_at: Wed, 11 Mar 2015 17:13:05 GMT
 - request:
     method: post
     uri: https://api.flickr.com/services/rest/
@@ -74,8 +74,8 @@ http_interactions:
       - FlickRaw/0.9.8
       Authorization:
       - OAuth realm="https://api.flickr.com/services/rest/", oauth_consumer_key="<FLICKR_API_KEY>",
-        oauth_nonce="092C0sA5RStztAm86VzUC4lS07gEfaNnCTwQz1TAY3Y%3D", oauth_signature="<FLICKR_SECRET>%26",
-        oauth_signature_method="PLAINTEXT", oauth_timestamp="1426006402", oauth_token="",
+        oauth_nonce="Wys2vnSmSwlJVK6t4GZKjntby0j9A6Al%2FLvTo3sEsdw%3D", oauth_signature="<FLICKR_SECRET>%26",
+        oauth_signature_method="PLAINTEXT", oauth_timestamp="1426093985", oauth_token="",
         oauth_version="1.0"
       Content-Type:
       - application/x-www-form-urlencoded
@@ -85,7 +85,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Tue, 10 Mar 2015 16:53:22 GMT
+      - Wed, 11 Mar 2015 17:13:05 GMT
       Content-Type:
       - application/json
       Content-Length:
@@ -97,14 +97,14 @@ http_interactions:
       Cache-Control:
       - private
       X-Served-By:
-      - www48.flickr.bf1.yahoo.com
+      - www47.flickr.bf1.yahoo.com
       Vary:
       - Accept-Encoding
       Age:
       - '0'
       Via:
-      - http/1.1 fts125.flickr.bf1.yahoo.com (ApacheTrafficServer/4.0.2 [cMsSf ]),
-        http/1.1 r02.ycpi.dea.yahoo.net (ApacheTrafficServer [cMsSf ])
+      - http/1.1 fts122.flickr.bf1.yahoo.com (ApacheTrafficServer/4.0.2 [cMsSf ]),
+        http/1.1 r10.ycpi.dea.yahoo.net (ApacheTrafficServer [cMsSf ])
       Server:
       - ATS
       Connection:
@@ -121,5 +121,5 @@ http_interactions:
         known copyright restrictions","url":"https:\/\/www.flickr.com\/commons\/usage\/"},{"id":"8","name":"United
         States Government Work","url":"http:\/\/www.usa.gov\/copyright.shtml"}]},"stat":"ok"}'
     http_version: 
-  recorded_at: Tue, 10 Mar 2015 16:53:22 GMT
+  recorded_at: Wed, 11 Mar 2015 17:13:05 GMT
 recorded_with: VCR 2.9.3