picky 2.7.0 → 3.0.0.pre1

data/lib/picky/sources/base.rb

@@ -1,87 +1,90 @@
-# = Data Sources
-#
-# Currently, Picky offers the following Sources:
-# * CSV (comma – or other – separated file)
-# * Couch (CouchDB, key-value store)
-# * DB (Databases, foremost MySQL)
-# * Delicious (http://del.icio.us, online bookmarking service)
-# See also:
-# http://github.com/floere/picky/wiki/Sources-Configuration
-#
-# Don't worry if your source isn't here. Adding your own is easy:
-# http://github.com/floere/picky/wiki/Contributing-sources
-#
-module Sources
+module Picky
 
-  # Sources are where your data comes from.
+  # = Data Sources
   #
-  # A source has 1 mandatory and 2 optional methods:
-  # * connect_backend (_optional_): called once for each type/category pair.
-  # * harvest: Used by the indexer to gather data. Yields an indexed_id (string or integer) and a string value.
-  # * take_snapshot (_optional_): called once for each index or category (if indexing a single category).
+  # Currently, Picky offers the following Sources:
+  # * CSV (comma – or other – separated file)
+  # * Couch (CouchDB, key-value store)
+  # * DB (Databases, foremost MySQL)
+  # * Delicious (http://del.icio.us, online bookmarking service)
+  # See also:
+  # http://github.com/floere/picky/wiki/Sources-Configuration
   #
-  # This base class "implements" all these methods, but they don't do anything.
-  # Subclass this class <tt>class MySource < Base</tt> and override the methods in your source to do something.
+  # Don't worry if your source isn't here. Adding your own is easy:
+  # http://github.com/floere/picky/wiki/Contributing-sources
   #
-  class Base
+  module Sources
 
-    attr_reader :key_format
-
-    # Connect to the backend.
+    # Sources are where your data comes from.
     #
-    # Called once per index/category combination before harvesting.
+    # A source has 1 mandatory and 2 optional methods:
+    # * connect_backend (_optional_): called once for each type/category pair.
+    # * harvest: Used by the indexer to gather data. Yields an indexed_id (string or integer) and a string value.
+    # * take_snapshot (_optional_): called once for each index or category (if indexing a single category).
     #
-    # Examples:
-    # * The DB backend connects the DB adapter.
-    # * We open a connection to a key value store.
-    # * We open an file with data.
+    # This base class "implements" all these methods, but they don't do anything.
+    # Subclass this class <tt>class MySource < Base</tt> and override the methods in your source to do something.
     #
-    def connect_backend
+    class Base
 
-    end
+      attr_reader :key_format
 
-    # Called by the indexer when gathering data.
-    #
-    # Yields the data (id, text for id) for the given category.
-    #
-    # When implementing or overriding your own,
-    # be sure to <tt>yield(id, text_for_id)</tt> (or <tt>block.call(id, text_for_id)</tt>)
-    # for the given type symbol and category symbol.
-    #
-    # Note: Since harvest needs to be implemented, it has no default impementation.
-    #
-    # def harvest category # :yields: id, text_for_id
-    #
-    # end
+      # Connect to the backend.
+      #
+      # Called once per index/category combination before harvesting.
+      #
+      # Examples:
+      # * The DB backend connects the DB adapter.
+      # * We open a connection to a key value store.
+      # * We open an file with data.
+      #
+      def connect_backend
 
-    # Used to take a snapshot of your data if it is fast changing.
-    #
-    # Called once for each index before harvesting.
-    # If it has been called on a source already by an index,
-    # it won't be called again for a category inside that index.
-    #
-    # Example:
-    # * In a DB source, a table based on the source's select statement is created.
-    #
-    def take_snapshot index
+      end
 
-    end
+      # Called by the indexer when gathering data.
+      #
+      # Yields the data (id, text for id) for the given category.
+      #
+      # When implementing or overriding your own,
+      # be sure to <tt>yield(id, text_for_id)</tt> (or <tt>block.call(id, text_for_id)</tt>)
+      # for the given type symbol and category symbol.
+      #
+      # Note: Since harvest needs to be implemented, it has no default impementation.
+      #
+      # def harvest category # :yields: id, text_for_id
+      #
+      # end
 
-    # Used to check if a snapshot has been done already.
-    #
-    # Example:
-    # * In a DB source, a table based on the source's select statement is created.
-    #
-    def with_snapshot index
-      connect_backend
-      @snapshot_taken ||= 0
-      if @snapshot_taken.zero?
-        timed_exclaim %Q{"#{index.identifier}": Taking snapshot of source data (if supported).}
-        take_snapshot index
+      # Used to take a snapshot of your data if it is fast changing.
+      #
+      # Called once for each index before harvesting.
+      # If it has been called on a source already by an index,
+      # it won't be called again for a category inside that index.
+      #
+      # Example:
+      # * In a DB source, a table based on the source's select statement is created.
+      #
+      def take_snapshot index
+
+      end
+
+      # Used to check if a snapshot has been done already.
+      #
+      # Example:
+      # * In a DB source, a table based on the source's select statement is created.
+      #
+      def with_snapshot index
+        connect_backend
+        @snapshot_taken ||= 0
+        if @snapshot_taken.zero?
+          take_snapshot index
+        end
+        @snapshot_taken += 1
+        yield
+        @snapshot_taken -= 1
       end
-      @snapshot_taken += 1
-      yield
-      @snapshot_taken -= 1
+
     end
 
   end

data/lib/picky/sources/couch.rb

@@ -1,72 +1,76 @@
-module Sources
+module Picky
 
-  # Raised when a Couch source is instantiated without a file.
-  #
-  # Example:
-  #   Sources::Couch.new(:column1, :column2) # without file option
-  #
-  class NoCouchDBGiven < StandardError; end
-
-  # A Couch database source.
-  #
-  # Options:
-  # * url
-  # and all the options of a <tt>RestClient::Resource</tt>.
-  # See http://github.com/archiloque/rest-client.
-  #
-  # Examples:
-  #  Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984')
-  #  Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984', user:'someuser', password:'somepassword')
-  #
-  class Couch < Base
+  module Sources
 
+    # A Couch database source.
+    #
+    # Options:
+    # * url
+    # and all the options of a <tt>RestClient::Resource</tt>.
+    # See http://github.com/archiloque/rest-client.
     #
+    # Examples:
+    #  Picky::Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984')
+    #  Picky::Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984', user:'someuser', password:'somepassword')
     #
-    def initialize *category_names, options
-      check_gem
+    class Couch < Base
 
-      Hash === options && options[:url] || raise_no_db_given(category_names)
+      # Raised when a Couch source is instantiated without a file.
+      #
+      # Example:
+      #   Picky::Sources::Couch.new(:column1, :column2) # without file option
+      #
+      class NoDBGiven < StandardError; end
 
-      @db = RestClient::Resource.new options.delete(:url), options
+      #
+      #
+      def initialize *category_names, options
+        check_gem
 
-      key_format   = options.delete :key_format
-      @key_format  = key_format && key_format.to_sym || :to_sym
-    end
+        Hash === options && options[:url] || raise_no_db_given(category_names)
 
-    def to_s
-      self.class.name
-    end
+        @db = RestClient::Resource.new options.delete(:url), options
 
-    # Tries to require the rest_client gem.
-    #
-    def check_gem # :nodoc:
-      require 'rest_client'
-    rescue LoadError
-      warn_gem_missing 'rest-client', 'the CouchDB source'
-      exit 1
-    end
+        key_format   = options.delete :key_format
+        @key_format  = key_format && key_format.to_sym || :to_sym
+      end
 
-    # Harvests the data to index.
-    #
-    # See important note, above.
-    #
-    @@id_key = '_id'
-    def harvest category
-      category_name = category.from.to_s
-      get_data do |doc|
-        yield doc[@@id_key], doc[category_name] || next
+      def to_s
+        self.class.name
       end
-    end
 
-    def get_data &block # :nodoc:
-      resp = @db['_all_docs?include_docs=true'].get
-      JSON.parse(resp)['rows'].
-        map{|row| row['doc']}.
-        each &block
-    end
+      # Tries to require the rest_client gem.
+      #
+      def check_gem # :nodoc:
+        require 'rest_client'
+      rescue LoadError
+        warn_gem_missing 'rest-client', 'the CouchDB source'
+        exit 1
+      end
+
+      # Harvests the data to index.
+      #
+      # See important note, above.
+      #
+      @@id_key = '_id'
+      def harvest category
+        category_name = category.from.to_s
+        get_data do |doc|
+          yield doc[@@id_key], doc[category_name] || next
+        end
+      end
 
-    def raise_no_db_given category_names # :nodoc:
-      raise NoCouchDBGiven.new(category_names.join(', '))
+      def get_data &block # :nodoc:
+        resp = @db['_all_docs?include_docs=true'].get
+        JSON.parse(resp)['rows'].
+          map{|row| row['doc']}.
+          each &block
+      end
+
+      def raise_no_db_given category_names # :nodoc:
+        raise NoDBGiven.new(category_names.join(', '))
+      end
     end
   end
-end
+
+end

data/lib/picky/sources/csv.rb

@@ -1,78 +1,82 @@
-module Sources
-
-  # Raised when a CSV source is instantiated without a file.
-  #
-  # Example:
-  #   Sources::CSV.new(:column1, :column2) # without file option
-  #
-  class NoCSVFileGiven < StandardError; end
-
-  # Describes a CSV source, a file with comma separated values in it.
-  #
-  # The first column is implicitly assumed to be the id column.
-  #
-  # It takes the same options as the Ruby 1.9 CSV class.
-  #
-  # Examples:
-  #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv')
-  #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', col_sep:';')
-  #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', row_sep:"\n")
-  #
-  class CSV < Base
-
-    # The CSV file's path, relative to PICKY_ROOT.
-    #
-    attr_reader :file_name
+module Picky
 
-    # The options that were passed into #new.
-    #
-    attr_reader :csv_options, :key_format
+  module Sources
 
-    # The data category names.
+    # Describes a CSV source, a file with comma separated values in it.
+    #
+    # The first column is implicitly assumed to be the id column.
     #
-    attr_reader :category_names
+    # It takes the same options as the Ruby 1.9 CSV class.
+    #
+    # Examples:
+    #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv')
+    #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', col_sep:';')
+    #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', row_sep:"\n")
+    #
+    class CSV < Base
 
-    def initialize *category_names, options
-      require 'csv'
-      @category_names = category_names
+      # Raised when a CSV source is instantiated without a file.
+      #
+      # Example:
+      #   Sources::CSV.new(:column1, :column2) # without file option
+      #
+      class NoFileGiven < StandardError; end
 
-      @csv_options = Hash === options && options || {}
-      @file_name   = @csv_options.delete(:file) || raise_no_file_given(category_names)
+      # The CSV file's path, relative to PICKY_ROOT.
+      #
+      attr_reader :file_name
 
-      key_format   = options.delete :key_format
-      @key_format  = key_format && key_format.to_sym || :to_i
-    end
+      # The options that were passed into #new.
+      #
+      attr_reader :csv_options, :key_format
 
-    def to_s
-      parameters = category_names
-      parameters << { file: file_name }
-      parameters << csv_options unless csv_options.empty?
-      %Q{#{self.class.name}(#{parameters.join(', ')})}
-    end
+      # The data category names.
+      #
+      attr_reader :category_names
 
-    # Raises a NoCSVFileGiven exception.
-    #
-    def raise_no_file_given category_names # :nodoc:
-      raise NoCSVFileGiven.new(category_names.join(', '))
-    end
+      def initialize *category_names, options
+        require 'csv'
+        @category_names = category_names
 
-    # Harvests the data to index.
-    #
-    def harvest category
-      index = category_names.index category.from
-      get_data do |ary|
-        indexed_id = ary.shift
-        text       = ary[index]
-        next unless text
-        text.force_encoding 'utf-8' # TODO Still needed?
-        yield indexed_id, text
+        @csv_options = Hash === options && options || {}
+        @file_name   = @csv_options.delete(:file) || raise_no_file_given(category_names)
+
+        key_format   = options.delete :key_format
+        @key_format  = key_format && key_format.to_sym || :to_i
+      end
+
+      def to_s
+        parameters = category_names
+        parameters << { file: file_name }
+        parameters << csv_options unless csv_options.empty?
+        %Q{#{self.class.name}(#{parameters.join(', ')})}
+      end
+
+      # Raises a NoCSVFileGiven exception.
+      #
+      def raise_no_file_given category_names # :nodoc:
+        raise NoFileGiven.new(category_names.join(', '))
+      end
+
+      # Harvests the data to index.
+      #
+      def harvest category
+        index = category_names.index category.from
+        get_data do |ary|
+          indexed_id = ary.shift
+          text       = ary[index]
+          next unless text
+          text.force_encoding 'utf-8' # TODO Still needed?
+          yield indexed_id, text
+        end
+      end
+
+      #
+      #
+      def get_data &block # :nodoc:
+        ::CSV.foreach file_name, csv_options, &block
       end
-    end
 
-    #
-    #
-    def get_data &block # :nodoc:
-      ::CSV.foreach file_name, csv_options, &block
     end
 
   end