purdie 0.0.11 → 0.0.12

data/features/vimeo-albums.feature

@@ -0,0 +1,27 @@
+@vcr
+Feature: Vimeo albums
+
+  @announce-stdout
+  Scenario: Generate YAML for a Vimeo album
+    Given a file named "_sources/vimeo" with:
+    """
+    https://vimeo.com/album/3296736
+    """
+    When I successfully run `purdie fetch`
+    Then a file named "_data/vimeo.yaml" should exist
+    And the file "_data/vimeo.yaml" should contain:
+    """
+    ---
+    - title: Safety On Board
+      id: 111356018
+      license: Attribution-NonCommercial-ShareAlike
+      license_url: http://creativecommons.org/licenses/by-nc-sa/3.0/
+    - title: Trapped In Hawaii
+      id: 110133117
+      license: Attribution-NonCommercial-ShareAlike
+      license_url: http://creativecommons.org/licenses/by-nc-sa/3.0/
+    - title: Discotheque Metamorphosis
+      id: 110132671
+      license: Attribution-NonCommercial-ShareAlike
+      license_url: http://creativecommons.org/licenses/by-nc-sa/3.0/
+    """

data/features/youtube-playlist.feature

@@ -0,0 +1,54 @@
+@vcr
+Feature: YouTube playlists
+
+@announce-stdout
+@announce-stderr
+  Scenario: Generate YAML for a YouTube playlist
+    Given a file named "_sources/youtube" with:
+    """
+    https://www.youtube.com/playlist?list=PLuPLM2FI60-OIgFTc9YCrGgH5XWGT6znV
+    """
+    When I successfully run `purdie fetch`
+    Then a file named "_data/youtube.yaml" should exist
+    And the file "_data/youtube.yaml" should contain:
+    """
+    ---
+    - title: Chuck Rainey - Bernard Purdie Project "Love the One Your With...Drink Muddy
+        Water"
+      id: U23Ezi6q30E
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Godfathers of Groove Bernard "Pretty" Purdie,Grant Green Jr,Ruben Wilson
+        March 30-31
+      id: _ZbWTg8G6eM
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Pretty Purdie - Good Livin' (Good Lovin')  HQ
+      id: zJnnNZk9o0Q
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Hot Popcorn   Bernard Purdie
+      id: baQe6MoSAHw
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Red Beans and Rice   Bernard Purdie
+      id: NLFP1T1e2BA
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Bernard Purdie - Ad Lib
+      id: E9E0WxLbqVA
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Bernard Purdie - Black Purd's Theme
+      id: wa-K4LouFVk
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: Bernard "Pretty" Purdie - Heavy Soul Slinger
+      id: LatmKZQd7-s
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    - title: 'Bernard "Pretty" Purdie: Funky Grooves in Japan'
+      id: P842kq0bnOc
+      license: YouTube
+      license_url: https://www.youtube.com/static?gl=GB&template=terms
+    """

data/lib/purdie/bernard.rb

@@ -15,6 +15,8 @@ module Purdie
       rescue Errno::ENOENT
         @sources = nil
       end
+
+      Purdie.debug @config.inspect
     end
 
     def source_file= path
@@ -26,6 +28,9 @@ module Purdie
     end
 
     def process list
+      bad_creds = []
+      bad_licenses = {}
+
       list.each do |line|
         next if line[0] == '#'
         next if line == ''
@@ -34,11 +39,29 @@ module Purdie
           print "Processing #{line.strip}... "
           grab line
         rescue NoMethodError => nme
-          puts "unrecognised URL [#{line}]" if nme.message == "undefined method `ingest' for nil:NilClass"
+          puts "unrecognised URL [#{line}]" if nme.status == "undefined method `ingest' for nil:NilClass"
+        rescue Purdie::LicenseException => le
+          bad_licenses[Purdie.basename le.service].push le.name rescue bad_licenses[Purdie.basename le.service] = [le.name]
+          puts 'fail'
+        rescue Purdie::CredentialsException => ce
+          bad_creds.push Purdie.basename(ce.service)
+          puts 'fail'
         else
           puts 'done'
         end
       end
+
+      if bad_creds.any?
+        raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{bad_creds.uniq.join ', '}"
+      end
+
+      if bad_licenses.any?
+        bad = bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
+        message = "Unknown licenses: #{bad}"
+        message += "\n"
+        message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
+        raise Purdie::PurdieException.new message
+      end
     end
 
     def fetch

data/lib/purdie/cli.rb

@@ -15,12 +15,16 @@ module Purdie
     def fetch
       b = Bernard.new
       b.source_file = options[:source_file] if options[:source_file]
+
       begin
         b.fetch
       rescue Exception => e
+        $stderr.puts "\n"
         $stderr.puts e.message
       #  exit 1
       end
     end
+
+    default_task :fetch
   end
 end

data/lib/purdie/exceptions.rb

@@ -0,0 +1,20 @@
+require 'purdie'
+
+module Purdie
+  class CredentialsException < Exception
+    attr_reader :service, :message
+
+    def initialize service, message
+      @service = service
+      @message = message
+    end
+  end
+
+  class PurdieException < Exception
+    attr_reader :message
+    
+    def initialize message
+      @message = message
+    end
+  end
+end

data/lib/purdie/helpers.rb

@@ -1,6 +1,6 @@
 module Purdie
   def Purdie.strip_scheme url
-    url.match(/http[s]?:\/\/(.*)/)[1]
+    url.match(/http[s]?:(.*)/)[1]
   end
 
   def Purdie.sanitise_url url
@@ -14,11 +14,26 @@ module Purdie
     when /\?.*v=/
         return CGI.parse(URI.parse(url).query)['v'].first
       else
-        Purdie.sanitise_url(url).split('/')[-1].to_i
+        sanitised = Purdie.sanitise_url url
+        parts = sanitised.split('/')
+        parts.reverse.each do |part|
+          next if ['in', 'photostream'].include? part
+          return part.to_i
+        end
     end
   end
 
   def Purdie.basename obj
+    if obj.class == Class
+      return obj.name.to_s.split('::').last
+    end
+
     obj.class.name.to_s.split('::').last
   end
+
+  def Purdie.debug message
+    File.open '../../wtf.log', 'w' do |f|
+      f.write message
+    end
+  end
 end

data/lib/purdie/ingester.rb

@@ -7,6 +7,8 @@ module Purdie
     INGESTERS = []
 
     def self.included base
+      # Voodoo: http://stackoverflow.com/questions/10692961/inheriting-class-methods-from-mixins
+      base.extend ClassMethods
       INGESTERS.push base
     end
 
@@ -14,7 +16,8 @@ module Purdie
       INGESTERS
     end
 
-    def initialize config
+    def initialize config = nil
+      config = Config.new unless config
       @config = config
       @items = []
 
@@ -22,17 +25,13 @@ module Purdie
     end
 
     def configure
-      @output_file = "#{Purdie.basename(self).downcase}.yaml"
-      # This still feels like such a hack
+      @output_file = "#{@config['output_dir']}/#{Purdie.basename(self).downcase}.yaml"
       specific_config = @config['services'][Purdie.basename self] rescue nil
 
       if specific_config
-        if specific_config['output_file']
-          @config['output_dir'] = File.dirname(specific_config['output_file'])
-          @output_file = File.basename(@config['services'][Purdie.basename self]['output_file'])
+        specific_config.each_pair do |key, value|
+          self.instance_variable_set("@#{key}", value)
         end
-
-        @size = specific_config['size'] if specific_config['size']
       end
     end
 
@@ -58,11 +57,17 @@ module Purdie
 
     def write
       if self.has_items?
-        FileUtils.mkdir_p @config['output_dir']
-        File.open "#{@config['output_dir']}/#{@output_file}", 'w' do |f|
+        FileUtils.mkdir_p File.dirname @output_file
+        File.open @output_file, 'w' do |f|
           f.write self.to_yaml
         end
       end
     end
+
+    module ClassMethods
+      def resolve url
+        [url]
+      end
+    end
   end
 end

data/lib/purdie/license_manager.rb

@@ -0,0 +1,41 @@
+require 'purdie'
+
+module Purdie
+  class LicenseManager
+    LOOKUPS = YAML.load File.read File.join(File.dirname(__FILE__), '..', '..', '_config/licenses.yaml')
+
+    def self.get service, raw_name
+      License.new service, raw_name, LOOKUPS[Purdie.basename service][raw_name]
+    end
+  end
+
+  class License
+    def initialize service, raw_name, values
+      raise LicenseException.new service, raw_name unless values
+      @values = values
+    end
+
+    def [] key
+      @values[key]
+    end
+
+    def method_missing method_name, *args
+      mname = method_name.to_s
+
+      if @values.include? mname
+        @values[mname]
+      else
+        raise NoMethodError
+      end
+    end
+  end
+
+  class LicenseException < Exception
+    attr_reader :service, :name
+
+    def initialize service, name
+      @service = service
+      @name = name
+    end
+  end
+end

data/lib/purdie/services/flickr.rb

@@ -21,7 +21,13 @@ module Purdie
       end
 
       def get url
-        flickr.photos.getInfo photo_id: Purdie.get_id(url)
+        begin
+          flickr.photos.getInfo photo_id: Purdie.get_id(url)
+        rescue FlickRaw::FlickrAppNotConfigured => fanc
+          raise Purdie::CredentialsException.new self, 'missing' if fanc.message == 'No API key or secret defined!'
+        rescue FlickRaw::FailedResponse => fr
+          raise Purdie::CredentialsException.new self, 'duff' if fr.message == "'flickr.photos.getInfo' - Invalid API Key (Key has invalid format)"
+        end
       end
 
       def distill url
@@ -32,12 +38,16 @@ module Purdie
         results['title'] = @config['default_title'] if photo['title'] == ''
         results['date'] = photo['dates']['taken'].split(' ')[0]
         results['photo_page'] = photo['urls'][0]['_content']
+      #  results['photo_page'] = url
         results['photo_url'] = FlickRaw.send(Flickr.url_for_size(@size), photo)
 
         license = licenses.select {|l| l['id'] == photo['license']}[0]
         results['license'] = license['name'].split(' License')[0]
         results['license_url'] = license['url']
 
+# WTF?
+#        results.attach_license self, photo['license']
+
         begin
           snapper = photo['tags'].select { |t| t['raw'] =~ /photographer/ }[0]
           results['photographer'] = snapper['raw'].split(':')[1]
@@ -82,7 +92,9 @@ module Purdie
         end
       end
 
-      def self.resolve_set url
+      def self.resolve url
+        return [url] unless url =~ /\/sets\//
+
         flickr.photosets.getPhotos(photoset_id: Purdie.get_id(url))['photo'].
           map { |member| member['id'] }.
           map { |id| flickr.photos.getInfo photo_id: id }.

data/lib/purdie/services/soundcloud.rb

@@ -11,19 +11,28 @@ module Purdie
       end
 
       def distill url
-        track = client.get '/resolve', url: url
+        begin
+          track = client.get '/resolve', url: url
+        rescue ArgumentError => ae
+          raise CredentialsException.new self, 'missing'
+        rescue ::SoundCloud::ResponseError => re
+          raise CredentialsException.new self, 'duff'
+        end
+
         results = {}
         results['title'] = track['title']
         results['id'] = track['id']
         results['location'] = track['description']
         results['date'] = "%4d-%02d-%02d" % [ track['release_year'], track['release_month'], track['release_day'] ]
-        results['license'] = @config['license_lookups'][track['license']]['full_name']
-        results['license_url'] = @config['license_lookups'][track['license']]['url']
+
+        results.attach_license self, track['license']
 
         results
       end
 
-      def self.resolve_set url
+      def self.resolve url
+        return [url] unless url =~ /\/sets\//
+
         client = ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
         client.get('/resolve', url: url).tracks.
           map { |track| track['permalink_url'] }

data/lib/purdie/services/vimeo.rb

@@ -9,17 +9,22 @@ module Purdie
         @id = Purdie.get_id url
         target = "#{Vimeo.host}/videos/#{@id}"
         response = HTTParty.get target, headers: Vimeo.headers
-        JSON.parse response.body
+        response = JSON.parse response.body
+        if response['error'] == 'A valid user token must be passed.'
+          raise CredentialsException.new self, 'missing and/or duff'
+        else
+          response
+        end
       end
 
       def distill url
         video = get url
-        results = {}
 
+        results = {}
         results['title'] = video['name']
         results['id'] = @id
-        results['license'] = @config['license_lookups'][video['license']]['full_name']
-        results['license_url'] = @config['license_lookups'][video['license']]['url']
+
+        results.attach_license self, video['license']
 
         results
       end
@@ -31,10 +36,12 @@ module Purdie
         }
       end
 
-      def self.resolve_set url
+      def self.resolve url
+        return [url] unless url =~ /\/albums?\//
+
         target = "#{Vimeo.host}/albums/#{Purdie.get_id url}/videos/"
         set = JSON.parse (HTTParty.get target, headers: Vimeo.headers).body
-        set['data'].map { |video| video['link'] }
+        set['data'].map { |video| video['uri'].sub '/videos', 'https://vimeo.com' }
       end
 
       def self.matcher

data/lib/purdie/services/youtube.rb

@@ -6,6 +6,9 @@ module Purdie
     class YouTube
       include Ingester
 
+      API_SERVICE_NAME = 'youtube'
+      API_VERSION = 'v3'
+
       def configure
         @api_service_name = 'youtube'
         @api_version = 'v3'
@@ -39,8 +42,13 @@ module Purdie
       end
 
       def get url
-        data = get_data Purdie.get_id(url), 'status,snippet'
-        JSON.parse data.body
+        begin
+          data = get_data Purdie.get_id(url), 'status,snippet'
+          JSON.parse data.body
+        rescue Google::APIClient::ClientError => ce
+          raise Purdie::CredentialsException.new self, 'missing' if ce.message.match /Daily Limit for Unauthenticated Use Exceeded/
+          raise Purdie::CredentialsException.new self, 'duff' if ce.message.match /Bad Request/
+        end
       end
 
       def distill url
@@ -49,12 +57,46 @@ module Purdie
 
         results['title'] = video['items'][0]['snippet']['localized']['title']
         results['id'] = Purdie.get_id(url)
-        results['license'] = @config['license_lookups'][video['items'][0]['status']['license']]['full_name']
-        results['license_url'] = @config['license_lookups'][video['items'][0]['status']['license']]['url']
+
+        results.attach_license self, video['items'][0]['status']['license']
 
         results
       end
 
+      def self.resolve url
+        query = CGI.parse(URI.parse(url).query).keys
+        return [url] unless (query.include?('list') and not query.include?('index'))
+
+        set = YouTube.client.execute!(
+          api_method: YouTube.yt_service.playlist_items.list,
+          parameters: {
+            playlistId: 'PLuPLM2FI60-OIgFTc9YCrGgH5XWGT6znV',
+            part: 'contentDetails',
+            maxResults: 50
+          }
+        ).body
+
+        ids = JSON.parse(set)['items'].map { |v| v['contentDetails']['videoId'] }
+
+        ids.map { |id| "https://youtube.com/watch?v=#{id}"}
+      end
+
+      def self.client
+        Google::APIClient.new(
+          key: ENV['YOUTUBE_API_KEY'],
+          authorization: nil,
+          application_name: self.class.name.split('::').first,
+          application_version: Purdie::VERSION
+        )
+      end
+
+      def self.yt_service
+        client.discovered_api(
+          API_SERVICE_NAME,
+          API_VERSION
+        )
+      end
+
       def self.matcher
         'youtube.com'
       end

data/lib/purdie/source_list.rb

@@ -5,20 +5,12 @@ module Purdie
     include Enumerable
 
     def initialize sources
-      s = sources
-      s = [sources] unless sources.class == Array
-      @sources = []
-      s.each do |source|
-        case source
-          when /sets/
-            @sources += SourceList.resolve_set source
-          else
-            @sources.push source
-        end
-      end
-
-      @sources.select! { |i| i !~ /^#/ }
-      @sources.uniq! { |item| Purdie.strip_scheme item }
+      @sources = [sources].
+        flatten.
+        select { |i| i !~ /^#/ }.
+        map { |source| SourceList.resolve source }.
+        flatten.
+        uniq { |item| Purdie.strip_scheme item }
     end
 
     def [] key
@@ -39,8 +31,10 @@ module Purdie
       SourceList.new File.readlines(source_file).map { |l| l.strip }
     end
 
-    def self.resolve_set source
-      Ingester.ingesters.select { |service| source =~ /#{service.matcher}/ }[0].resolve_set source
+    def self.resolve source
+      service_class = Ingester.ingesters.select { |service| source =~ /#{service.matcher}/ }[0]
+      return [] unless service_class
+      service_class.resolve source
     end
   end
 end

data/lib/purdie/version.rb

@@ -1,3 +1,3 @@
 module Purdie
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

data/lib/purdie.rb

@@ -12,6 +12,8 @@ require 'purdie/bernard'
 require 'purdie/config'
 require 'purdie/ingester'
 require 'purdie/source_list'
+require 'purdie/license_manager'
+require 'purdie/exceptions'
 
 require 'purdie/services/soundcloud'
 require 'purdie/services/flickr'
@@ -19,3 +21,11 @@ require 'purdie/services/vimeo'
 require 'purdie/services/youtube'
 
 Dotenv.load
+
+class Hash
+  def attach_license service, license
+    l = Purdie::LicenseManager.get service, license
+    self['license'] = l['full_name']
+    self['license_url'] = l['url']
+  end
+end

data/spec/duff_keys_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+
+module Purdie
+  module Services
+    describe Flickr do
+      before :each do
+        unset_env
+      end
+
+      after :each do
+        reset_env
+        FlickRaw.api_key = ENV['FLICKR_API_KEY']
+        FlickRaw.shared_secret = ENV['FLICKR_SECRET']
+      end
+
+      it 'responds usefully in the face of no credentials' do
+        FlickRaw.api_key = nil
+        FlickRaw.shared_secret = nil
+        f = Flickr.new
+###        expect { f.distill 'https://www.flickr.com/photos/rawfunkmaharishi/15631479625/' }.to raise_exception { |e|
+###          expect(e).to be_a Purdie::CredentialsException
+###          expect(e.message).to eq 'Flickr credentials missing'
+###        }
+      end
+
+      it 'responds usefully in the face of duff credentials' do
+        FlickRaw.api_key = 'abc'
+        FlickRaw.shared_secret = '123'
+        f = Flickr.new
+###        expect { f.distill 'https://www.flickr.com/photos/rawfunkmaharishi/15631479625/' }.to raise_exception { |e|
+###          expect(e).to be_a Purdie::CredentialsException
+###          expect(e.message).to eq 'Flickr credentials might be duff'
+###        }
+      end
+    end
+
+    describe SoundCloud do
+      after :each do
+        reset_env
+      end
+
+      it 'responds usefully in the face of no credentials' do
+        unset_env
+        s = SoundCloud.new
+        expect { s.distill 'https://soundcloud.com/rawfunkmaharishi/bernard' }.to raise_exception { |e|
+          expect(e).to be_a Purdie::CredentialsException
+          expect(e.service.class).to eq Purdie::Services::SoundCloud
+          expect(e.message).to eq 'missing'
+        }
+      end
+
+      it 'responds usefully in the face of duff credentials' do
+        randomise_env
+        s = SoundCloud.new
+        expect { s.distill 'https://soundcloud.com/rawfunkmaharishi/bernard' }.to raise_exception { |e|
+          expect(e).to be_a Purdie::CredentialsException
+          expect(e.service.class).to eq Purdie::Services::SoundCloud
+          expect(e.message).to eq 'duff'
+        }
+      end
+    end
+
+    describe Vimeo do
+      after :each do
+        reset_env
+      end
+
+      it 'responds usefully in the face of no credentials' do
+        unset_env
+        v = Vimeo.new
+        expect { v.distill 'https://vimeo.com/111356018' }.to raise_exception { |e|
+          expect(e).to be_a Purdie::CredentialsException
+          expect(e.service.class).to eq Purdie::Services::Vimeo
+          expect(e.message).to eq 'missing and/or duff'
+        }
+      end
+
+      it 'responds usefully in the face of duff credentials' do
+        randomise_env
+        v = Vimeo.new
+          expect { v.distill 'https://vimeo.com/111356018' }.to raise_exception { |e|
+          expect(e).to be_a Purdie::CredentialsException
+          expect(e.service.class).to eq Purdie::Services::Vimeo
+          expect(e.message).to eq 'missing and/or duff'
+        }
+      end
+    end
+
+    describe YouTube do
+      after :each do
+        reset_env
+      end
+
+      it 'responds usefully in the face of no credentials' do
+        unset_env
+        y = YouTube.new
+        expect { y.distill 'https://www.youtube.com/watch?v=JCix1XW329g' }.to raise_exception { |e|
+          expect(e).to be_a Purdie::CredentialsException
+          expect(e.service.class).to eq Purdie::Services::YouTube
+          expect(e.message).to eq 'missing'
+        }
+      end
+
+      it 'responds usefully in the face of duff credentials' do
+        randomise_env
+        y = YouTube.new
+        expect { y.distill 'https://www.youtube.com/watch?v=JCix1XW329g' }.to raise_exception { |e|
+          expect(e).to be_a Purdie::CredentialsException
+          expect(e.service.class).to eq Purdie::Services::YouTube
+          expect(e.message).to eq 'duff'
+        }
+      end
+    end
+  end
+end