purdie 0.0.12 → 0.0.13

data/features/support/hooks.rb

@@ -2,3 +2,11 @@ Before do
   FileUtils.mkdir_p "tmp/aruba/_config/"
   FileUtils.cp "#{$fixtures}/_config/purdie.yaml", "tmp/aruba/_config/"
 end
+
+Before "@clearconf" do
+  FileUtils.rm "tmp/aruba/_config/purdie.yaml"
+end
+
+After do
+  FileUtils.rm "tmp/aruba/_config/purdie.yaml"
+end

data/features/youtube-playlist.feature

@@ -1,8 +1,6 @@
 @vcr
 Feature: YouTube playlists
 
-@announce-stdout
-@announce-stderr
   Scenario: Generate YAML for a YouTube playlist
     Given a file named "_sources/youtube" with:
     """

data/lib/purdie.rb

@@ -10,10 +10,12 @@ require 'purdie/version'
 require 'purdie/helpers'
 require 'purdie/bernard'
 require 'purdie/config'
+require 'purdie/item'
 require 'purdie/ingester'
 require 'purdie/source_list'
 require 'purdie/license_manager'
 require 'purdie/exceptions'
+require 'purdie/resolver'
 
 require 'purdie/services/soundcloud'
 require 'purdie/services/flickr'
@@ -21,11 +23,3 @@ require 'purdie/services/vimeo'
 require 'purdie/services/youtube'
 
 Dotenv.load
-
-class Hash
-  def attach_license service, license
-    l = Purdie::LicenseManager.get service, license
-    self['license'] = l['full_name']
-    self['license_url'] = l['url']
-  end
-end

data/lib/purdie/bernard.rb

@@ -5,7 +5,7 @@ module Purdie
     attr_reader :config
 
     def initialize
-      @config = Config.new
+      @config = Config.instance.config
       begin
         @sources = Dir.entries(@config['source_dir']).select { |e|
           e !~ /^\./
@@ -15,71 +15,18 @@ module Purdie
       rescue Errno::ENOENT
         @sources = nil
       end
-
-      Purdie.debug @config.inspect
-    end
-
-    def source_file= path
-      @sources = [path]
-    end
-
-    def services
-      @services ||= Ingester.ingesters.map { |i| i.new @config }
     end
 
-    def process list
-      bad_creds = []
-      bad_licenses = {}
-
-      list.each do |line|
-        next if line[0] == '#'
-        next if line == ''
-
-        begin
-          print "Processing #{line.strip}... "
-          grab line
-        rescue NoMethodError => nme
-          puts "unrecognised URL [#{line}]" if nme.status == "undefined method `ingest' for nil:NilClass"
-        rescue Purdie::LicenseException => le
-          bad_licenses[Purdie.basename le.service].push le.name rescue bad_licenses[Purdie.basename le.service] = [le.name]
-          puts 'fail'
-        rescue Purdie::CredentialsException => ce
-          bad_creds.push Purdie.basename(ce.service)
-          puts 'fail'
-        else
-          puts 'done'
-        end
-      end
-
-      if bad_creds.any?
-        raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{bad_creds.uniq.join ', '}"
-      end
-
-      if bad_licenses.any?
-        bad = bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
-        message = "Unknown licenses: #{bad}"
-        message += "\n"
-        message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
-        raise Purdie::PurdieException.new message
-      end
-    end
+#    def source_file= path
+#      @sources = path
+#    end
 
     def fetch
-      raise Exception.new 'No data sources specified' unless @sources
+      raise PurdieException.new 'No data sources specified' unless @sources
 
-      @sources.each do |source|
-        process SourceList.from_file source
+      @sources.map { |s| SourceList.from_file s }.each do |source|
+        source.write
       end
-
-      dump
-    end
-
-    def grab url
-      services.select{ |s| url =~ /#{s.class.matcher}/ }[0].ingest url
-    end
-
-    def dump
-      services.map { |service| service.write }
     end
   end
 end

data/lib/purdie/cli.rb

@@ -14,7 +14,7 @@ module Purdie
                   :desc => 'Specify source file to process'
     def fetch
       b = Bernard.new
-      b.source_file = options[:source_file] if options[:source_file]
+#      b.source_file = options[:source_file] if options[:source_file]
 
       begin
         b.fetch

data/lib/purdie/config.rb

@@ -1,22 +1,31 @@
-require 'purdie'
+require 'singleton'
 
 module Purdie
   class Config
+    include Singleton
+
     def initialize
-      @conf = YAML.load File.read File.join(File.dirname(__FILE__), '..', '..', '_config/defaults.yaml')
+      reset!
+    end
+
+    def reset! # testing a singleton is hard
+      @config = OpenStruct.new fetch_yaml 'defaults'
 
       if File.exists? '_config/purdie.yaml'
-        y = YAML.load File.read '_config/purdie.yaml'
-        @conf.deep_merge! y
+        @local = OpenStruct.new YAML.load File.read '_config/purdie.yaml'
+
+        @config = OpenStruct.new (@config.to_h.deep_merge @local)
       end
     end
 
-    def [] key
-      @conf[key]
+    def config
+      @config
     end
 
-    def []= key, value
-      @conf[key] = value
+    private
+
+    def fetch_yaml file
+      YAML.load(File.open(File.join(File.dirname(__FILE__), '..', '..', '_config/%s.yaml' % file)))
     end
   end
 end

data/lib/purdie/helpers.rb

@@ -37,3 +37,11 @@ module Purdie
     end
   end
 end
+
+class Hash
+  def attach_license service, license
+    l = Purdie::LicenseManager.get service, license
+    self['license'] = l['full_name']
+    self['license_url'] = l['url']
+  end
+end

data/lib/purdie/ingester.rb

@@ -4,20 +4,22 @@ module Purdie
   module Ingester
     attr_reader :config, :subconfig, :matcher
 
-    INGESTERS = []
+    include Enumerable
+
+    class << self
+      attr_reader :ingesters
+    end
+
+    @ingesters = []
 
     def self.included base
       # Voodoo: http://stackoverflow.com/questions/10692961/inheriting-class-methods-from-mixins
       base.extend ClassMethods
-      INGESTERS.push base
-    end
-
-    def self.ingesters
-      INGESTERS
+      @ingesters.push base
     end
 
     def initialize config = nil
-      config = Config.new unless config
+      config = Config.instance.config unless config
       @config = config
       @items = []
 
@@ -39,16 +41,14 @@ module Purdie
       @items.push distill url
     end
 
-    def [] key
-      @items[key]
-    end
-
-    def count
-      @items.count
+    def each
+      @items.each do |item|
+        yield item
+      end
     end
 
-    def has_items?
-      @items.count > 0
+    def [] key
+      @items[key]
     end
 
     def to_yaml
@@ -56,7 +56,7 @@ module Purdie
     end
 
     def write
-      if self.has_items?
+      if self.any?
         FileUtils.mkdir_p File.dirname @output_file
         File.open @output_file, 'w' do |f|
           f.write self.to_yaml

data/lib/purdie/item.rb

@@ -0,0 +1,36 @@
+module Purdie
+  class Item
+    attr_reader :url, :parent_file, :datas
+
+    def initialize url
+      @url = url
+      @datas = {}
+    end
+
+    def distill
+      @datas = service.distill @url
+    end
+
+    def []= key, value
+      @datas[key] = value
+    end
+
+    def [] key
+      @datas[key]
+    end
+
+    def service
+      c = Ingester.ingesters.select { |s| url =~ /#{s.matcher}/ }.first
+      c.new
+    end
+
+    def to_yaml
+      Purdie.debug 'wtf'
+      @datas.to_yaml
+    end
+
+    def source_list= source_list
+      @parent_file = source_list.parent_file
+    end
+  end
+end

data/lib/purdie/resolver.rb

@@ -0,0 +1,15 @@
+module Purdie
+  class Resolver
+    def self.resolve urls
+      urls = [urls].flatten
+      resolved = []
+      
+      urls.each do |url|
+        service_class = Ingester.ingesters.select { |service| url =~ /#{service.matcher}/ }[0]
+        resolved += service_class.resolve(url) if service_class
+      end
+
+      resolved.uniq { |url| Purdie.strip_scheme url }
+    end
+  end
+end

data/lib/purdie/services/soundcloud.rb

@@ -19,7 +19,7 @@ module Purdie
           raise CredentialsException.new self, 'duff'
         end
 
-        results = {}
+        results = {} 
         results['title'] = track['title']
         results['id'] = track['id']
         results['location'] = track['description']

data/lib/purdie/source_list.rb

@@ -4,13 +4,11 @@ module Purdie
   class SourceList
     include Enumerable
 
+    attr_reader :items
+    attr_accessor :parent_file
+
     def initialize sources
-      @sources = [sources].
-        flatten.
-        select { |i| i !~ /^#/ }.
-        map { |source| SourceList.resolve source }.
-        flatten.
-        uniq { |item| Purdie.strip_scheme item }
+      @sources = Resolver.resolve(sources).map { |item| Item.new item }
     end
 
     def [] key
@@ -19,22 +17,62 @@ module Purdie
 
     def each &block
       @sources.each do |source|
-        if block_given?
-          block.call source
-        else
-          yield source
+        yield source
+      end
+    end
+
+    def process
+      bad_creds = []
+      bad_licenses = {}
+      @items = []
+      @sources.each do |source|
+        begin
+          print "Processing #{source.url}..."
+          source.distill
+          puts 'done'
+        rescue Purdie::CredentialsException => ce
+          bad_creds.push Purdie.basename(ce.service)
+          puts 'fail'
+        end
+        @items.push source
+
+        if bad_creds.any?
+          raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{bad_creds.uniq.join ', '}"
+        end
+
+        if bad_licenses.any?
+          bad = bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
+          message = "Unknown licenses: #{bad}"
+          message += "\n"
+          message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
+          raise Purdie::PurdieException.new message
         end
       end
     end
 
-    def self.from_file source_file
-      SourceList.new File.readlines(source_file).map { |l| l.strip }
+    def write
+      process
+
+      FileUtils.mkdir_p File.dirname output_file
+      File.open output_file, 'w' do |f|
+        dump = @items.map { |item| item.datas }
+        f.write dump.to_yaml
+      end
+    end
+
+    def output_file
+      base = File.basename(parent_file).split '.'
+      base.push nil if base.count == 1
+      base[-1] = 'yaml'
+
+      File.join ['_data', base.join('.')]
     end
 
-    def self.resolve source
-      service_class = Ingester.ingesters.select { |service| source =~ /#{service.matcher}/ }[0]
-      return [] unless service_class
-      service_class.resolve source
+    def self.from_file source_file
+      sl = SourceList.new File.readlines(source_file).map { |l| l.strip }.select { |i| i !~ /^#/ }
+      sl.parent_file = source_file
+
+      sl
     end
   end
 end

data/lib/purdie/version.rb

@@ -1,3 +1,3 @@
 module Purdie
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 end

data/purdie.gemspec

@@ -18,25 +18,25 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'httparty', '~> 0.13'
-  spec.add_dependency 'deep_merge', '~> 1.0'
-  spec.add_dependency 'dotenv', '~> 2.0'
-  spec.add_dependency 'flickraw-cached', '= 20120701'
-  spec.add_dependency 'soundcloud'
-  spec.add_dependency 'activesupport'
-  spec.add_dependency 'google-api-client'
+  spec.add_dependency 'thor',              '~> 0.19'
+  spec.add_dependency 'httparty',          '~> 0.13'
+  spec.add_dependency 'deep_merge',        '~> 1.0'
+  spec.add_dependency 'dotenv',            '~> 2.0'
+  spec.add_dependency 'flickraw-cached',   '= 20120701'
+  spec.add_dependency 'soundcloud',        '~> 0.3'
+  spec.add_dependency 'activesupport',     '~> 4.2'
+  spec.add_dependency 'google-api-client', '~> 0.8'
 
-  spec.add_development_dependency 'bundler', '~> 1.7'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.2'
-  spec.add_development_dependency 'aruba', '~> 0.5'
-  spec.add_development_dependency 'guard', '~> 2.12'
-  spec.add_development_dependency 'guard-rspec', '~> 4.5'
-  spec.add_development_dependency 'guard-cucumber', '~> 1.5'
+  spec.add_development_dependency 'bundler',                 '~> 1.7'
+  spec.add_development_dependency 'rake',                    '~> 10.0'
+  spec.add_development_dependency 'rspec',                   '~> 3.2'
+  spec.add_development_dependency 'aruba',                   '~> 0.5'
+  spec.add_development_dependency 'guard',                   '~> 2.12'
+  spec.add_development_dependency 'guard-rspec',             '~> 4.5'
+  spec.add_development_dependency 'guard-cucumber',          '~> 1.5'
   spec.add_development_dependency 'terminal-notifier-guard', '~> 1.6'
-  spec.add_development_dependency 'coveralls', '~> 0.7'
-  spec.add_development_dependency 'webmock', '~> 1.20'
-  spec.add_development_dependency 'vcr', '~> 2.9'
-  spec.add_development_dependency 'timecop', '~> 0.7'
+  spec.add_development_dependency 'coveralls',               '~> 0.7'
+  spec.add_development_dependency 'webmock',                 '~> 1.20'
+  spec.add_development_dependency 'vcr',                     '~> 2.9'
+  spec.add_development_dependency 'timecop',                 '~> 0.7'
 end