purdie 0.0.13 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f65941fa63765aacc68d10628f57455119d907ff
-  data.tar.gz: 663003c250b66bed25aad0967c9d770bd6c7890b
+  metadata.gz: 47daab89a649c37ffa1b983b4fe593be8117d31f
+  data.tar.gz: bfeab60a60e4765da990339e500e6d206119b1e0
 SHA512:
-  metadata.gz: f0ed75ddf476ad8c33942148d48b11e1ebde71e827d45c36987bcc51f0440b001ab603278b45b4091613ae4560904a06156463a2e50353ea3172b03b6de26de6
-  data.tar.gz: e123f933c514079cb7cd5ddcb23d987f9605e97bbecd5ca4f4e477ef628e2e89b4300b768be378a79ca0a64b199973d4b1118122b825646bbba40a1383c182e7
+  metadata.gz: fd5342cbf21b9af2b75b1dde23f86475d11c704426f5aed5098afceb55135b1ab0a54a01d9cffbe92529896316639371e432b794daa2ca30236d281c5c914085
+  data.tar.gz: 405010930c759d7da43155e0338bf5b2a33413f010f987029692192a2a7aefec4f16899dfde2e49114d11235145692950d17b533e13347521518408872ed1935

data/README.md

@@ -43,7 +43,7 @@ You need to create a *_sources* directory in your Jekyll project, containing fil
     https://soundcloud.com/rawfunkmaharishi/hexaflexagon-1
     https://soundcloud.com/rawfunkmaharishi/junalbandi-3
 
-It also resolves sets/albums on Flickr and SoundCloud (and soon Vimeo), so this kind of thing will work:
+It also resolves sets/albums on all of the supported services, so this kind of thing will work:
 
     https://www.flickr.com/photos/pikesley/sets/72157648589429938/
 
@@ -110,12 +110,12 @@ Tread carefully for now, because my metadata hacks aren't fully documented, and
 
 There's no reason I couldn't support other services - I've already added [YouTube](https://github.com/rawfunkmaharishi/purdie/blob/master/spec/services/youtube_spec.rb) [support](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/youtube.rb) and others should be fairly simple. There's some introspection magic at the heart of all of this which means that as long as each service is represented by a class that:
 
-* includes the `Purdie::Ingester` module, and
+* includes the `Purdie::Service` module, and
 * sports a `::matcher` class method which returns a string which will pick a URL out of an input file, and
 * has a `#distill` method which takes a URL representing an item on the service and returns a hash of metadata, see e.g.
-  * [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb#L33)
-  * [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb#L13)
-  * [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb#L20)
+  * [Flickr](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/flickr.rb)
+  * [SoundCloud](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/soundcloud.rb)
+  * [Vimeo](https://github.com/rawfunkmaharishi/purdie/blob/master/lib/purdie/services/vimeo.rb)
 * and optionally a `::resolve` class method which takes a set or album URL for the service and returns a list of URLs for individual items
 
 then this should all Just Work. There's definitely a blog post in this, because Ruby introspection and metaprogramming is just mind-bogglingly powerful (and dangerous).

data/lib/purdie.rb

@@ -11,15 +11,13 @@ require 'purdie/helpers'
 require 'purdie/bernard'
 require 'purdie/config'
 require 'purdie/item'
-require 'purdie/ingester'
 require 'purdie/source_list'
 require 'purdie/license_manager'
 require 'purdie/exceptions'
 require 'purdie/resolver'
 
+require 'purdie/service'
 require 'purdie/services/soundcloud'
 require 'purdie/services/flickr'
 require 'purdie/services/vimeo'
 require 'purdie/services/youtube'
-
-Dotenv.load

data/lib/purdie/bernard.rb

@@ -1,8 +1,6 @@
-require 'purdie'
-
 module Purdie
   class Bernard
-    attr_reader :config
+    attr_accessor :verbose
 
     def initialize
       @config = Config.instance.config
@@ -24,7 +22,7 @@ module Purdie
     def fetch
       raise PurdieException.new 'No data sources specified' unless @sources
 
-      @sources.map { |s| SourceList.from_file s }.each do |source|
+      @sources.map { |s| SourceList.from_file s, @verbose }.each do |source|
         source.write
       end
     end

data/lib/purdie/cli.rb

@@ -14,6 +14,7 @@ module Purdie
                   :desc => 'Specify source file to process'
     def fetch
       b = Bernard.new
+      b.verbose = true
 #      b.source_file = options[:source_file] if options[:source_file]
 
       begin

data/lib/purdie/exceptions.rb

@@ -1,5 +1,3 @@
-require 'purdie'
-
 module Purdie
   class CredentialsException < Exception
     attr_reader :service, :message
@@ -12,7 +10,7 @@ module Purdie
 
   class PurdieException < Exception
     attr_reader :message
-    
+
     def initialize message
       @message = message
     end

data/lib/purdie/item.rb

@@ -1,6 +1,6 @@
 module Purdie
   class Item
-    attr_reader :url, :parent_file, :datas
+    attr_reader :url, :datas
 
     def initialize url
       @url = url
@@ -20,7 +20,7 @@ module Purdie
     end
 
     def service
-      c = Ingester.ingesters.select { |s| url =~ /#{s.matcher}/ }.first
+      c = Service.services.select { |s| url =~ /#{s.matcher}/ }.first
       c.new
     end
 
@@ -28,9 +28,5 @@ module Purdie
       Purdie.debug 'wtf'
       @datas.to_yaml
     end
-
-    def source_list= source_list
-      @parent_file = source_list.parent_file
-    end
   end
 end

data/lib/purdie/license_manager.rb

@@ -1,5 +1,3 @@
-require 'purdie'
-
 module Purdie
   class LicenseManager
     LOOKUPS = YAML.load File.read File.join(File.dirname(__FILE__), '..', '..', '_config/licenses.yaml')

data/lib/purdie/resolver.rb

@@ -3,9 +3,9 @@ module Purdie
     def self.resolve urls
       urls = [urls].flatten
       resolved = []
-      
+
       urls.each do |url|
-        service_class = Ingester.ingesters.select { |service| url =~ /#{service.matcher}/ }[0]
+        service_class = Service.services.select { |service| url =~ /#{service.matcher}/ }[0]
         resolved += service_class.resolve(url) if service_class
       end
 

data/lib/purdie/{ingester.rb → service.rb}

@@ -1,27 +1,22 @@
 Dotenv.load
 
 module Purdie
-  module Ingester
-    attr_reader :config, :subconfig, :matcher
-
-    include Enumerable
-
+  module Service
     class << self
-      attr_reader :ingesters
+      attr_reader :services
     end
 
-    @ingesters = []
+    @services = []
 
     def self.included base
       # Voodoo: http://stackoverflow.com/questions/10692961/inheriting-class-methods-from-mixins
       base.extend ClassMethods
-      @ingesters.push base
+      @services.push base
     end
 
     def initialize config = nil
       config = Config.instance.config unless config
       @config = config
-      @items = []
 
       configure
     end
@@ -37,33 +32,6 @@ module Purdie
       end
     end
 
-    def ingest url
-      @items.push distill url
-    end
-
-    def each
-      @items.each do |item|
-        yield item
-      end
-    end
-
-    def [] key
-      @items[key]
-    end
-
-    def to_yaml
-      @items.to_yaml
-    end
-
-    def write
-      if self.any?
-        FileUtils.mkdir_p File.dirname @output_file
-        File.open @output_file, 'w' do |f|
-          f.write self.to_yaml
-        end
-      end
-    end
-
     module ClassMethods
       def resolve url
         [url]

data/lib/purdie/services/flickr.rb

@@ -1,4 +1,3 @@
-require 'purdie'
 require 'flickraw-cached'
 
 FlickRaw.api_key = ENV['FLICKR_API_KEY']
@@ -9,7 +8,7 @@ module Purdie
     class Flickr
       attr_accessor :size
 
-      include Purdie::Ingester
+      include Purdie::Service
 
       def configure
         @size = 240 # pixels
@@ -37,8 +36,7 @@ module Purdie
         results['title'] = photo['title']
         results['title'] = @config['default_title'] if photo['title'] == ''
         results['date'] = photo['dates']['taken'].split(' ')[0]
-        results['photo_page'] = photo['urls'][0]['_content']
-      #  results['photo_page'] = url
+        results['photo_page'] = url
         results['photo_url'] = FlickRaw.send(Flickr.url_for_size(@size), photo)
 
         license = licenses.select {|l| l['id'] == photo['license']}[0]

data/lib/purdie/services/soundcloud.rb

@@ -1,10 +1,9 @@
-require 'purdie'
 require 'soundcloud'
 
 module Purdie
   module Services
     class SoundCloud
-      include Purdie::Ingester
+      include Purdie::Service
 
       def client
         @client ||= ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
@@ -19,7 +18,7 @@ module Purdie
           raise CredentialsException.new self, 'duff'
         end
 
-        results = {} 
+        results = {}
         results['title'] = track['title']
         results['id'] = track['id']
         results['location'] = track['description']

data/lib/purdie/services/vimeo.rb

@@ -1,9 +1,7 @@
-require 'purdie'
-
 module Purdie
   module Services
     class Vimeo
-      include Purdie::Ingester
+      include Purdie::Service
 
       def get url
         @id = Purdie.get_id url

data/lib/purdie/services/youtube.rb

@@ -1,10 +1,9 @@
-require 'purdie'
 require 'google/api_client'
 
 module Purdie
   module Services
     class YouTube
-      include Ingester
+      include Service
 
       API_SERVICE_NAME = 'youtube'
       API_VERSION = 'v3'

data/lib/purdie/source_list.rb

@@ -1,52 +1,60 @@
-require 'purdie'
-
 module Purdie
   class SourceList
     include Enumerable
 
     attr_reader :items
-    attr_accessor :parent_file
+    attr_accessor :parent_file, :verbose
 
     def initialize sources
       @sources = Resolver.resolve(sources).map { |item| Item.new item }
+      @items = []
+      @bad_creds = []
+      @bad_licenses = {}
     end
 
     def [] key
       @sources[key]
     end
 
-    def each &block
+    def each
       @sources.each do |source|
         yield source
       end
     end
 
     def process
-      bad_creds = []
-      bad_licenses = {}
-      @items = []
       @sources.each do |source|
         begin
-          print "Processing #{source.url}..."
+          print "Processing #{source.url}... " if @verbose
           source.distill
-          puts 'done'
+          puts 'done' if @verbose
         rescue Purdie::CredentialsException => ce
-          bad_creds.push Purdie.basename(ce.service)
-          puts 'fail'
+          @bad_creds.push Purdie.basename(ce.service)
+          puts 'fail' if @verbose
+        rescue Purdie::LicenseException => le
+          @bad_licenses[Purdie.basename le.service].push le.name rescue @bad_licenses[Purdie.basename le.service] = [le.name]
+          puts 'fail' if @verbose
         end
         @items.push source
 
-        if bad_creds.any?
-          raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{bad_creds.uniq.join ', '}"
-        end
+        bad_credentials?
+        bad_licenses?
+      end
+    end
 
-        if bad_licenses.any?
-          bad = bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
-          message = "Unknown licenses: #{bad}"
-          message += "\n"
-          message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
-          raise Purdie::PurdieException.new message
-        end
+    def bad_credentials?
+      if @bad_creds.any?
+        raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{@bad_creds.uniq.join ', '}"
+      end
+    end
+
+    def bad_licenses?
+      if @bad_licenses.any?
+        bad = @bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
+        message = "Unknown licenses: #{bad}"
+        message += "\n"
+        message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
+        raise Purdie::PurdieException.new message
       end
     end
 
@@ -68,10 +76,10 @@ module Purdie
       File.join ['_data', base.join('.')]
     end
 
-    def self.from_file source_file
+    def self.from_file source_file, verbose = nil
       sl = SourceList.new File.readlines(source_file).map { |l| l.strip }.select { |i| i !~ /^#/ }
       sl.parent_file = source_file
-
+      sl.verbose = verbose
       sl
     end
   end

data/lib/purdie/version.rb

@@ -1,3 +1,3 @@
 module Purdie
-  VERSION = "0.0.13"
+  VERSION = "0.1.0"
 end

data/spec/{bernard_spec.rb → purdie/bernard_spec.rb}

@@ -2,6 +2,12 @@ require 'spec_helper'
 
 module Purdie
   describe Bernard do
+    it 'does not throw a fit when initialized with no _sources dir' do
+      FileUtils.rmdir File.join(File.dirname(__FILE__), '..', '..', '_sources')
+      b = Bernard.new
+      expect {b.fetch}.to raise_error
+    end
+
     it 'processes files', :vcr do
       FileUtils.cp 'spec/support/fixtures/soundcloud.sounds', '_sources/'
       FileUtils.cp 'spec/support/fixtures/vimeo.vids', '_sources/'

data/spec/{item_spec.rb → purdie/item_spec.rb}

@@ -27,12 +27,6 @@ bass: Rainey
       )
     end
 
-    it 'knows what file it came from' do
-      sl = instance_double('SourceList', parent_file: 'some/path/bernard.purdie')
-      i.source_list = sl
-      expect(i.parent_file).to eq 'some/path/bernard.purdie'
-    end
-
     it 'knows what service it belongs to' do
       expect(i.service.class).to eq Purdie::Services::SoundCloud
     end

data/spec/{services → purdie/services}/flickr_spec.rb

@@ -35,8 +35,8 @@ photographer: kim
       end
 
       it 'ingests data for a photo without a specific photographer tag', :vcr do
-        @f.ingest 'https://www.flickr.com/photos/pikesley/16649739916/'
-        expect(@f[0].to_yaml).to match /photographer: sam/
+        dist = @f.distill 'https://www.flickr.com/photos/pikesley/16649739916/'
+        expect(dist.to_yaml).to match /photographer: sam/
       end
 
       it 'falls back to the default photographer name', :vcr do
@@ -76,8 +76,8 @@ photographer: kim
 
       context 'do the local config lookups' do
         it 'looks up the default title', :vcr do
-          @f.ingest 'https://www.flickr.com/photos/cluttercup/16393865760/'
-          expect(@f[0]['title']).to eq 'Raw Funk Maharishi'
+          dist = @f.distill 'https://www.flickr.com/photos/cluttercup/16393865760/'
+          expect(dist['title']).to eq 'Raw Funk Maharishi'
         end
       end
     end

data/spec/{services → purdie/services}/soundcloud_spec.rb

@@ -25,20 +25,6 @@ license_url: http://creativecommons.org/licenses/by-nc-sa/4.0/
         )
       end
 
-      it 'ingests a track', :vcr do
-        @sc.ingest 'https://soundcloud.com/rawfunkmaharishi/bernard'
-        expect(@sc[0].to_yaml).to eq(
-"---
-title: Bernard
-id: 192841052
-location: Islington Academy
-date: '2015-02-18'
-license: Attribution-NonCommercial-ShareAlike
-license_url: http://creativecommons.org/licenses/by-nc-sa/4.0/
-"
-        )
-      end
-
       context 'resolve a set' do
         it 'resolves a set from a url', :vcr do
           set = SoundCloud.resolve 'https://soundcloud.com/rawfunkmaharishi/sets/islington-academy-sessions'