purdie 0.0.9 → 0.0.10

data/features/support/vcr.rb

@@ -11,7 +11,9 @@ VCR.configure do |c|
   [
     'SOUNDCLOUD_CLIENT_ID',
     'FLICKR_API_KEY',
-    'FLICKR_SECRET'
+    'FLICKR_SECRET',
+    'VIMEO_BEARER_TOKEN',
+    'YOUTUBE_API_KEY'
   ].each do |env_var|
     c.filter_sensitive_data("<#{env_var}>") { ENV[env_var] }
   end
@@ -22,7 +24,7 @@ VCR.cucumber_tags do |t|
 end
 
 class VcrFriendlyMain
-  def initialize(argv, stdin, stdout, stderr, kernel)
+  def initialize argv, stdin, stdout, stderr, kernel
     @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
   end
 
@@ -30,16 +32,16 @@ class VcrFriendlyMain
     $stdin = @stdin
     $stdout = @stdout
     $stderr = @stderr
-    Purdie::CLI.start(@argv)
+    Purdie::CLI.start @argv
   end
 end
 
-Before('@vcr') do
+Before '@vcr' do
   Aruba::InProcess.main_class = VcrFriendlyMain
   Aruba.process = Aruba::InProcess
 end
 
-After('@vcr') do
+After '@vcr' do
   Aruba.process = Aruba::SpawnProcess
   VCR.eject_cassette
   $stdin = STDIN

data/features/youtube.feature

@@ -0,0 +1,20 @@
+@vcr
+Feature: YouTube
+
+  Scenario: Generate YouTube YAML
+    Given a file named "_sources/youtube.movies" with:
+    """
+    https://www.youtube.com/watch?v=Qt_J0jNqtZg&list=PLuPLM2FI60-M0-aWejF9WgB-Dkt1TuQXv&index=2
+    """
+    When I successfully run `purdie fetch`
+    Then a file named "_data/youtube.yaml" should exist
+    And the file "_data/youtube.yaml" should contain:
+    """
+    - title: Sam Pikesley – Vandalising your Github commit history for fun and profit
+      id: Qt_J0jNqtZg
+      license: Attribution
+      license_url: https://creativecommons.org/licenses/by/3.0/
+    """
+    And a file named "_data/soundcloud.yaml" should not exist
+    And a file named "_data/vimeo.yaml" should not exist
+    And a file named "_data/flickr.yaml" should not exist

data/lib/purdie/bernard.rb

@@ -7,37 +7,55 @@ module Purdie
     def initialize
       @config = Config.new
       begin
-        @sources = Dir.entries(@config['source_dir']).select { |e| e !~ /^\./ }
-        @sources.map! { |s| "#{@config['source_dir']}/#{s}"}
+        @sources = Dir.entries(@config['source_dir']).select { |e|
+          e !~ /^\./
+        }.map { |s|
+          "#{@config['source_dir']}/#{s}"
+        }
       rescue Errno::ENOENT
         @sources = nil
       end
     end
 
-    def source_file path
+    def source_file= path
       @sources = [path]
     end
 
+    def services
+      @services ||= Ingester.ingesters.map { |i| i.new @config }
+    end
+
+    def process list
+      list.each do |line|
+        next if line[0] == '#'
+        next if line == ''
+
+        begin
+          print "Processing #{line.strip}... "
+          grab line
+        rescue NoMethodError => nme
+          puts "unrecognised URL [#{line}]" if nme.message == "undefined method `ingest' for nil:NilClass"
+        else
+          puts 'done'
+        end
+      end
+    end
+
     def fetch
       raise Exception.new 'No data sources specified' unless @sources
 
-      services = Ingester.includees.map { |i| i.new @config }
-
       @sources.each do |source|
-        File.readlines(source).each do |line|
-          next if line[0] == '#'
-
-          begin
-            print "Processing #{line.strip}... "
-            services.select{ |s| line =~ /#{s.matcher}/ }[0].ingest line
-          rescue NoMethodError => nme
-            puts 'unrecognised URL' if nme.message == "undefined method `ingest' for nil:NilClass"
-          else
-            puts 'done'
-          end
-        end
+        process SourceList.from_file source
       end
 
+      dump
+    end
+
+    def grab url
+      services.select{ |s| url =~ /#{s.class.matcher}/ }[0].ingest url
+    end
+
+    def dump
       services.map { |service| service.write }
     end
   end

data/lib/purdie/cli.rb

@@ -14,7 +14,7 @@ module Purdie
                   :desc => 'Specify source file to process'
     def fetch
       b = Bernard.new
-      b.source_file options[:source_file] if options[:source_file]
+      b.source_file = options[:source_file] if options[:source_file]
       begin
         b.fetch
       rescue Exception => e

data/lib/purdie/helpers.rb

@@ -0,0 +1,24 @@
+module Purdie
+  def Purdie.strip_scheme url
+    url.match(/http[s]?:\/\/(.*)/)[1]
+  end
+
+  def Purdie.sanitise_url url
+    url.strip!
+    url = url[0..-2] if url[-1] == '/'
+    url
+  end
+
+  def Purdie.get_id url
+    case url
+    when /\?.*v=/
+        return CGI.parse(URI.parse(url).query)['v'].first
+      else
+        Purdie.sanitise_url(url).split('/')[-1].to_i
+    end
+  end
+
+  def Purdie.basename obj
+    obj.class.name.to_s.split('::').last
+  end
+end

data/lib/purdie/ingester.rb

@@ -4,14 +4,14 @@ module Purdie
   module Ingester
     attr_reader :config, :subconfig, :matcher
 
-    INCLUDEES = []
+    INGESTERS = []
 
     def self.included base
-      INCLUDEES.push base
+      INGESTERS.push base
     end
 
-    def self.includees
-      INCLUDEES
+    def self.ingesters
+      INGESTERS
     end
 
     def initialize config
@@ -22,14 +22,25 @@ module Purdie
     end
 
     def configure
-      if @config['services']
-        if @config['services'][Purdie.basename self]
-          if @config['services'][Purdie.basename self]['output_file']
-            @config['output_dir'] = File.dirname(@config['services'][Purdie.basename self]['output_file'])
-            @output_file = File.basename(@config['services'][Purdie.basename self]['output_file'])
-          end
+      @output_file = "#{Purdie.basename(self).downcase}.yaml"
+      # This still feels like such a hack
+      specific_config = @config['services'][Purdie.basename self] rescue nil
+
+      if specific_config
+        if specific_config['output_file']
+          @config['output_dir'] = File.dirname(specific_config['output_file'])
+          @output_file = File.basename(@config['services'][Purdie.basename self]['output_file'])
         end
+
+        @size = specific_config['size'] if specific_config['size']
       end
+
+
+      # Can I do something like
+      # @config['services'][Purdie.basename self].each do |c|
+      #   @#{c} = c
+      # end
+      # ?
     end
 
     def ingest url
@@ -40,6 +51,10 @@ module Purdie
       @items[key]
     end
 
+    def count
+      @items.count
+    end
+
     def has_items?
       @items.count > 0
     end

data/lib/purdie/services/flickr.rb

@@ -8,12 +8,12 @@ FlickRaw.shared_secret = ENV['FLICKR_SECRET']
 module Purdie
   module Services
     class Flickr
+      attr_accessor :size
+
       include Purdie::Ingester
 
       def configure
-        @matcher = 'flickr.com'
-        @output_file = 'flickr.yaml'
-
+        @size = 240 # pixels
         super
       end
 
@@ -33,7 +33,7 @@ module Purdie
         results['title'] = @config['default_title'] if photo['title'] == ''
         results['date'] = photo['dates']['taken'].split(' ')[0]
         results['photo_page'] = photo['urls'][0]['_content']
-        results['photo_url'] = FlickRaw.url_m(photo)
+        results['photo_url'] = FlickRaw.send(Flickr.url_for_size(@size), photo)
 
         license = licenses.select {|l| l['id'] == photo['license']}[0]
         results['license'] = license['name'].split(' License')[0]
@@ -43,7 +43,11 @@ module Purdie
           snapper = photo['tags'].select { |t| t['raw'] =~ /photographer/ }[0]
           results['photographer'] = snapper['raw'].split(':')[1]
         rescue NoMethodError
-          results['photographer'] = @config['photographer_lookups'][photo['owner']['username']]
+          if @config['photographer_lookups']
+            results['photographer'] = @config['photographer_lookups'][photo['owner']['username']]
+          else
+            results['photographer'] = photo['owner']['username']
+          end
           unless results['photographer']
             results['photographer'] = photo['owner']['username']
           end
@@ -51,6 +55,45 @@ module Purdie
 
         results
       end
+
+      def self.url_for_size size = nil
+        size = 500 unless size
+
+        case
+          when size <= 75
+            return :url_s
+          when size <= 100
+            return :url_t
+          when size <= 150
+            return :url_q
+          when size <= 240
+            return :url_m
+          when size <= 320
+            return :url_n
+          when size <= 500
+            return :url
+          when size <= 640
+            return :url_z
+          when size <= 800
+            return :url_c
+          when size <= 1024
+            return :url_b
+          else
+            return :url_o
+        end
+      end
+
+      def self.resolve_set url
+        flickr.photosets.getPhotos(photoset_id: Purdie.get_id(url))['photo'].
+          map { |member| member['id'] }.
+          map { |id| flickr.photos.getInfo photo_id: id }.
+          map { |picture| picture['urls'][0]['_content'] }
+      end
+
+
+      def self.matcher
+        'flickr.com'
+      end
     end
   end
 end

data/lib/purdie/services/soundcloud.rb

@@ -7,30 +7,12 @@ module Purdie
     class SoundCloud
       include Purdie::Ingester
 
-      def configure
-        @host = 'https://api.soundcloud.com'
-        @matcher = 'soundcloud.com'
-        @output_file = 'soundcloud.yaml'
-
-        super
-      end
-
-      def all_tracks
-        @all_tracks ||= begin
-          url = "#{@host}/users/#{ENV['SOUNDCLOUD_USER_ID']}/tracks?client_id=#{ENV['SOUNDCLOUD_CLIENT_ID']}"
-          response = HTTParty.get url
-          JSON.parse response.body
-        end
-      end
-
-      def get url
-        all_tracks.select do |track|
-          Purdie.strip_scheme(track['permalink_url']) == Purdie.strip_scheme(url)
-        end[0]
+      def client
+        @client ||= ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
       end
 
       def distill url
-        track = get url
+        track = client.get '/resolve', url: url
         results = {}
         results['title'] = track['title']
         results['id'] = track['id']
@@ -41,6 +23,16 @@ module Purdie
 
         results
       end
+
+      def self.matcher
+        'soundcloud.com'
+      end
+
+      def self.resolve_set url
+        client = ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
+        client.get('/resolve', url: url).tracks.
+          map { |track| track['permalink_url'] }
+      end
     end
   end
 end

data/lib/purdie/services/vimeo.rb

@@ -7,9 +7,6 @@ module Purdie
 
       def configure
         @host = 'https://api.vimeo.com'
-        @matcher = 'vimeo.com'
-        @output_file = 'vimeo.yaml'
-
         super
       end
 
@@ -37,6 +34,10 @@ module Purdie
 
         results
       end
+        
+      def self.matcher
+        'vimeo.com'
+      end
     end
   end
 end

data/lib/purdie/services/youtube.rb

@@ -0,0 +1,64 @@
+require 'purdie'
+require 'cgi'
+require 'google/api_client'
+
+module Purdie
+  module Services
+    class YouTube
+      include Ingester
+
+      def configure
+        @api_service_name = 'youtube'
+        @api_version = 'v3'
+        super
+      end
+
+      def client
+        @client ||= Google::APIClient.new(
+            key: ENV['YOUTUBE_API_KEY'],
+            authorization: nil,
+            application_name: self.class.name.split('::').first,
+            application_version: Purdie::VERSION
+          )
+      end
+
+      def yt_service
+        @yt_service ||= client.discovered_api(
+          @api_service_name,
+          @api_version
+        )
+      end
+
+      def get_data id, part
+        client.execute!(
+          api_method: yt_service.videos.list,
+          parameters: {
+            id: id,
+            part: part
+          }
+        )
+      end
+
+      def get url
+        data = get_data Purdie.get_id(url), 'status,snippet'
+        JSON.parse data.body
+      end
+
+      def distill url
+        video = get url
+        results = {}
+
+        results['title'] = video['items'][0]['snippet']['localized']['title']
+        results['id'] = Purdie.get_id(url)
+        results['license'] = @config['license_lookups'][video['items'][0]['status']['license']]['full_name']
+        results['license_url'] = @config['license_lookups'][video['items'][0]['status']['license']]['url']
+
+        results
+      end
+
+      def self.matcher
+        'youtube.com'
+      end
+    end
+  end
+end

data/lib/purdie/source_list.rb

@@ -0,0 +1,46 @@
+require 'purdie'
+
+module Purdie
+  class SourceList
+    include Enumerable
+
+    def initialize sources
+      s = sources
+      s = [sources] unless sources.class == Array
+      @sources = []
+      s.each do |source|
+        case source
+          when /sets/
+            @sources += SourceList.resolve_set source
+          else
+            @sources.push source
+        end
+      end
+
+      @sources.select! { |i| i !~ /^#/ }
+      @sources.uniq! { |item| Purdie.strip_scheme item }
+    end
+
+    def [] key
+      @sources[key]
+    end
+
+    def each &block
+      @sources.each do |source|
+        if block_given?
+          block.call source
+        else
+          yield source
+        end
+      end
+    end
+
+    def self.from_file source_file
+      SourceList.new File.readlines(source_file).map { |l| l.strip }
+    end
+
+    def self.resolve_set source
+      Ingester.ingesters.select { |service| source =~ /#{service.matcher}/ }[0].resolve_set source
+    end
+  end
+end

data/lib/purdie/version.rb

@@ -1,3 +1,3 @@
 module Purdie
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end

data/lib/purdie.rb

@@ -3,35 +3,18 @@ require 'yaml'
 require 'deep_merge'
 require 'httparty'
 require 'dotenv'
+require 'soundcloud'
 require 'flickraw-cached'
 require 'active_support/inflector'
 
 require 'purdie/version'
+require 'purdie/helpers'
 require 'purdie/bernard'
 require 'purdie/config'
 require 'purdie/ingester'
+require 'purdie/source_list'
 
 require 'purdie/services/soundcloud'
 require 'purdie/services/flickr'
 require 'purdie/services/vimeo'
-
-module Purdie
-  def Purdie.strip_scheme url
-    url.match(/http[s]?:\/\/(.*)/)[1]
-  end
-
-  def Purdie.sanitise_url url
-    url.strip!
-    url = url[0..-2] if url[-1] == '/'
-
-    url
-  end
-
-  def Purdie.get_id url
-    Purdie.sanitise_url(url).split('/')[-1].to_i
-  end
-
-  def Purdie.basename obj
-    obj.class.name.to_s.split('::').last
-  end
-end
+require 'purdie/services/youtube'

data/list

@@ -0,0 +1 @@
+https://soundcloud.com/rawfunkmaharishi/bernard

data/purdie.gemspec

@@ -21,9 +21,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'httparty', '~> 0.13'
   spec.add_dependency 'deep_merge', '~> 1.0'
-  spec.add_dependency 'dotenv', '~> 1.0'
+  spec.add_dependency 'dotenv', '~> 2.0'
   spec.add_dependency 'flickraw-cached', '= 20120701'
+  spec.add_dependency 'soundcloud'
   spec.add_dependency 'activesupport'
+  spec.add_dependency 'google-api-client'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/spec/helpers_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+module Purdie
+  describe Bernard do
+    it 'strips a scheme' do
+      expect(Purdie.strip_scheme 'http://foo.bar/stuff').to eq 'foo.bar/stuff'
+      expect(Purdie.strip_scheme 'https://bar.foo/stuff').to eq 'bar.foo/stuff'
+    end
+
+    it 'sanitises a URL' do
+      expect(Purdie.sanitise_url 'http://foo.bar/stuff/').to eq 'http://foo.bar/stuff'
+    end
+
+    it 'gets an id' do
+      expect(Purdie.get_id 'http://foo.bar/654321').to eq 654321
+      expect(Purdie.get_id 'https://www.youtube.com/watch?v=JCix1XW329g').to eq 'JCix1XW329g'
+    end
+
+    it 'gets a basename for a class' do
+      module Medeski
+        module Martin
+          class Wood
+          end
+        end
+      end
+
+      @mmw = Medeski::Martin::Wood.new
+      expect(Purdie.basename @mmw).to eq 'Wood'
+    end
+  end
+end

data/spec/purdie_spec.rb

@@ -12,29 +12,5 @@ module Purdie
       b = Bernard.new
       expect {b.fetch}.to raise_error
     end
-
-    it 'strips a scheme' do
-      expect(Purdie.strip_scheme 'http://foo.bar/stuff').to eq 'foo.bar/stuff'
-      expect(Purdie.strip_scheme 'https://bar.foo/stuff').to eq 'bar.foo/stuff'
-    end
-
-    it 'sanitises a URL' do
-      expect(Purdie.sanitise_url 'http://foo.bar/stuff/').to eq 'http://foo.bar/stuff'
-    end
-
-    it 'gets an id' do
-      expect(Purdie.get_id 'http://foo.bar/654321').to eq 654321
-    end
-
-    it 'gets a basename for a class' do
-      module Medeski
-        module Martin
-          class Wood
-          end
-        end
-      end
-      @mmw = Medeski::Martin::Wood.new
-      expect(Purdie.basename @mmw).to eq 'Wood'
-    end
   end
 end

data/spec/services/flickr_spec.rb

@@ -37,7 +37,39 @@ photographer: jane
         )
       end
 
-      it 'falls back to the default photographer name', :vcr do        expect(@f.distill('https://www.flickr.com/photos/pikesley/16649739916/')['photographer']).to eq 'pikesley'
+      it 'falls back to the default photographer name', :vcr do
+        expect(@f.distill('https://www.flickr.com/photos/pikesley/16649739916/')['photographer']).to eq 'pikesley'
+      end
+
+      context 'FlickRaw url_ methods' do
+        it 'gives the plain (500px) method by default' do
+          expect(Flickr.url_for_size).to eq :url
+        end
+
+        it 'gives the 75px url when we ask for 20 pixels' do
+          expect(Flickr.url_for_size 20).to eq :url_s
+        end
+
+        it 'gives the 640px url when we ask for 600 pixels' do
+          expect(Flickr.url_for_size 600).to eq :url_z
+        end
+
+        it 'gives the 500px url when we ask for it' do
+          expect(Flickr.url_for_size 500).to eq :url
+        end
+
+        it 'gives the biggest url when we ask for something larger' do
+          expect(Flickr.url_for_size 6000000).to eq :url_o
+        end
+      end
+
+      context 'resolve a set' do
+        it 'resolves a set from a url', :vcr do
+          set = Flickr.resolve_set 'https://www.flickr.com/photos/pikesley/sets/72157649827363868/'
+          expect(set.count).to eq 8
+          expect(set[0]).to eq 'https://www.flickr.com/photos/pikesley/16252009191/'
+          expect(set[7]).to eq 'https://www.flickr.com/photos/pikesley/16752239531/'
+        end
       end
     end
   end