picky 2.7.0 → 3.0.0.pre1

data/lib/picky/sources/db.rb

@@ -1,171 +1,175 @@
-module Sources
-
-  # Describes a database source. Needs a SELECT statement
-  # (with id in it), and a file option or the options from an AR config file.
-  #
-  # The select statement can be as complicated as you want,
-  # as long as it has an id in it and as long as it can be
-  # used in a CREATE TABLE AS statement.
-  # (working on that last one)
-  #
-  # Examples:
-  #  Sources::DB.new('SELECT id, title, author, year FROM books') # Uses the config from app/db.yml by default.
-  #  Sources::DB.new('SELECT id, title, author, year FROM books', file: 'app/some_db.yml')
-  #  Sources::DB.new('SELECT b.id, b.title, b.author, b.publishing_year as year FROM books b INNER JOIN ON ...', file: 'app/some_db.yml')
-  #  Sources::DB.new('SELECT id, title, author, year FROM books', adapter: 'mysql', host:'localhost', ...)
-  #
-  class DB < Base
-
-    # The select statement that was passed in.
-    #
-    attr_reader :select_statement
+module Picky
 
-    # The database adapter.
-    #
-    attr_reader :database
+  module Sources
 
-    # The database connection options that were either passed in or loaded from the given file.
+    # Describes a database source. Needs a SELECT statement
+    # (with id in it), and a file option or the options from an AR config file.
     #
-    attr_reader :connection_options, :options
-
-    @@traversal_id = :__picky_id
-
-    def initialize select_statement, options = { file: 'app/db.yml' }
-      @select_statement = select_statement
-      @database         = create_database_adapter
-      @options          = options
-    end
+    # The select statement can be as complicated as you want,
+    # as long as it has an id in it and as long as it can be
+    # used in a CREATE TABLE AS statement.
+    # (working on that last one)
+    #
+    # Examples:
+    #  Sources::DB.new('SELECT id, title, author, year FROM books') # Uses the config from app/db.yml by default.
+    #  Sources::DB.new('SELECT id, title, author, year FROM books', file: 'app/some_db.yml')
+    #  Sources::DB.new('SELECT b.id, b.title, b.author, b.publishing_year as year FROM books b INNER JOIN ON ...', file: 'app/some_db.yml')
+    #  Sources::DB.new('SELECT id, title, author, year FROM books', adapter: 'mysql', host:'localhost', ...)
+    #
+    class DB < Base
 
-    def to_s
-      parameters = [select_statement.inspect]
-      parameters << options unless options.empty?
-      %Q{#{self.class.name}(#{parameters.join(', ')})}
-    end
+      # The select statement that was passed in.
+      #
+      attr_reader :select_statement
 
-    # Creates a database adapter for use with this source.
-    def create_database_adapter # :nodoc:
-      # TODO Do not use ActiveRecord directly.
+      # The database adapter.
       #
-      # TODO Use set_table_name etc.
+      attr_reader :database
+
+      # The database connection options that were either passed in or loaded from the given file.
       #
-      adapter_class = Class.new ActiveRecord::Base
-      adapter_class.abstract_class = true
-      adapter_class
-    end
+      attr_reader :connection_options, :options
 
-    # Configure the backend.
-    #
-    # Options:
-    #  Either
-    #  * file => 'some/filename.yml' # With an active record configuration.
-    #  Or
-    #  * The configuration as a hash.
-    #
-    def configure options # :nodoc:
-      @connection_options = if filename = options[:file]
-        File.open(File.join(PICKY_ROOT, filename)) { |file| YAML::load(file) }
-      else
-        options
+      @@traversal_id = :__picky_id
+
+      def initialize select_statement, options = { file: 'app/db.yml' }
+        @select_statement = select_statement
+        @database         = create_database_adapter
+        @options          = options
       end
-      self
-    end
 
-    # Connect the backend.
-    #
-    # Will raise unless connection options have been given.
-    #
-    def connect_backend
-      configure @options
-      raise "Database backend not configured" unless connection_options
-      database.establish_connection connection_options
-    end
+      def to_s
+        parameters = [select_statement.inspect]
+        parameters << options unless options.empty?
+        %Q{#{self.class.name}(#{parameters.join(', ')})}
+      end
 
-    # Take a snapshot of the data.
-    #
-    # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
-    #
-    def take_snapshot index
-      origin = snapshot_table_name index.name
-      on_database = database.connection
+      # Creates a database adapter for use with this source.
+      def create_database_adapter # :nodoc:
+        # TODO Do not use ActiveRecord directly. Use set_table_name etc.
+        #
+        adapter_class = Class.new ActiveRecord::Base
+        adapter_class.abstract_class = true
+        adapter_class
+      end
 
-      # Drop the table if it exists.
+      # Configure the backend.
       #
-      on_database.drop_table origin if on_database.table_exists?(origin)
-
-      # The adapters currently do not support this.
+      # Options:
+      #  Either
+      #  * file => 'some/filename.yml' # With an active record configuration.
+      #  Or
+      #  * The configuration as a hash.
       #
-      on_database.execute "CREATE TABLE #{origin} AS #{select_statement}"
+      def configure options # :nodoc:
+        @connection_options = if filename = options[:file]
+          File.open(File.join(PICKY_ROOT, filename)) { |file| YAML::load(file) }
+        else
+          options
+        end
+        self
+      end
 
-      # Add a column that Picky uses to traverse the table's entries.
+      # Connect the backend.
       #
-      on_database.add_column origin, @@traversal_id, :primary_key, :null => :false
+      # Will raise unless connection options have been given.
+      #
+      def connect_backend
+        configure @options
+        raise "Database backend not configured" unless connection_options
+        database.establish_connection connection_options
+      end
 
-      # Execute any special queries this index needs executed.
+      # Take a snapshot of the data.
       #
-      on_database.execute index.after_indexing if index.after_indexing
-    end
+      # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
+      #
+      def take_snapshot index
+        timed_exclaim %Q{"#{index.identifier}": Taking snapshot of database data.}
 
-    # Counts all the entries that are used for the index.
-    #
-    def count index_name
-      database.connection.select_value("SELECT COUNT(#{@@traversal_id}) FROM #{snapshot_table_name(index_name)}").to_i
-    end
+        origin = snapshot_table_name index.name
+        on_database = database.connection
 
-    # The name of the snapshot table created by Picky.
-    #
-    def snapshot_table_name index_name
-      "picky_#{index_name}_index"
-    end
+        # Drop the table if it exists.
+        #
+        on_database.drop_table origin if on_database.table_exists?(origin)
 
-    # Harvests the data to index in chunks.
-    #
-    def harvest category, &block
-      (0..count(category.index_name)).step(chunksize) do |offset|
-        get_data category, offset, &block
+        # The adapters currently do not support this.
+        #
+        on_database.execute "CREATE TABLE #{origin} AS #{select_statement}"
+
+        # Add a column that Picky uses to traverse the table's entries.
+        #
+        on_database.add_column origin, @@traversal_id, :primary_key, :null => :false
+
+        # Execute any special queries this index needs executed.
+        #
+        on_database.execute index.after_indexing if index.after_indexing
       end
-    end
 
-    # Gets the data from the backend.
-    #
-    def get_data category, offset, &block # :nodoc:
-      select_statement = harvest_statement_with_offset category, offset
+      # Counts all the entries that are used for the index.
+      #
+      def count index_name
+        database.connection.select_value("SELECT COUNT(#{@@traversal_id}) FROM #{snapshot_table_name(index_name)}").to_i
+      end
 
-      # TODO Rewrite ASAP.
+      # The name of the snapshot table created by Picky.
       #
-      if database.connection.adapter_name == "PostgreSQL"
-        id_key   = 'id'
-        text_key = category.from.to_s
-        database.connection.execute(select_statement).each do |hash|
-          id, text = hash.values_at id_key, text_key
-          yield id, text if text
+      def snapshot_table_name index_name
+        "picky_#{index_name}_index"
+      end
+
+      # Harvests the data to index in chunks.
+      #
+      def harvest category, &block
+        (0..count(category.index_name)).step(chunksize) do |offset|
+          get_data category, offset, &block
         end
-      else
-        database.connection.execute(select_statement).each do |id, text|
-          yield id, text if text
+      end
+
+      # Gets the data from the backend.
+      #
+      def get_data category, offset, &block # :nodoc:
+        select_statement = harvest_statement_with_offset category, offset
+
+        # TODO Rewrite ASAP.
+        #
+        if database.connection.adapter_name == "PostgreSQL"
+          id_key   = 'id'
+          text_key = category.from.to_s
+          database.connection.execute(select_statement).each do |hash|
+            id, text = hash.values_at id_key, text_key
+            yield id, text if text
+          end
+        else
+          database.connection.execute(select_statement).each do |id, text|
+            yield id, text if text
+          end
         end
       end
-    end
 
-    # Builds a harvest statement for getting data to index.
-    #
-    def harvest_statement_with_offset category, offset
-      statement = harvest_statement category
+      # Builds a harvest statement for getting data to index.
+      #
+      def harvest_statement_with_offset category, offset
+        statement = harvest_statement category
 
-      statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
+        statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
 
-      "#{statement} st.#{@@traversal_id} > #{offset} LIMIT #{chunksize}"
-    end
+        "#{statement} st.#{@@traversal_id} > #{offset} LIMIT #{chunksize}"
+      end
 
-    # The harvest statement used to pull data from the snapshot table.
-    #
-    def harvest_statement category
-      "SELECT id, #{category.from} FROM #{snapshot_table_name(category.index_name)} st"
-    end
+      # The harvest statement used to pull data from the snapshot table.
+      #
+      def harvest_statement category
+        "SELECT id, #{category.from} FROM #{snapshot_table_name(category.index_name)} st"
+      end
+
+      # The amount of records that are loaded each chunk.
+      #
+      def chunksize
+        25_000
+      end
 
-    # The amount of records that are loaded each chunk.
-    #
-    def chunksize
-      25_000
     end
 
   end

data/lib/picky/sources/delicious.rb

@@ -1,57 +1,61 @@
-module Sources
-
-  # Describes a Delicious (http://deli.cio.us) source.
-  #
-  # This source has a fixed set of categories:
-  # * title
-  # * tags
-  # * url
-  #
-  # Examples:
-  #  Sources::CSV.new('usrnam', 'paswrd')
-  #
-  class Delicious < Base
-
-    def initialize username, password
-      check_gem
-      @username = username
-      @password = password
-    end
-    def check_gem # :nodoc:
-      require 'www/delicious'
-    rescue LoadError
-      warn_gem_missing 'www-delicious', 'the delicious source'
-      exit 1
-    end
+module Picky
 
-    def to_s
-      "#{self.class.name}(#{@username})"
-    end
+  module Sources
 
-    # Harvests the data to index.
+    # Describes a Delicious (http://deli.cio.us) source.
     #
-    def harvest category
-      get_data do |indexed_id, data|
-        text = data[category.from]
-        next unless text
-        yield indexed_id, text
-      end
-    end
-
+    # This source has a fixed set of categories:
+    # * title
+    # * tags
+    # * url
     #
+    # Examples:
+    #  Sources::CSV.new('usrnam', 'paswrd')
     #
-    def get_data # :nodoc:
-      @generated_id ||= 0
-      @posts ||= WWW::Delicious.new(@username, @password).posts_recent(count: 100)
-      @posts.each do |post|
-        data = {
-          title: post.title,
-          tags:  post.tags.join(' '),
-          url:   post.url.to_s
-        }
-        @generated_id += 1
-        yield @generated_id, data
+    class Delicious < Base
+
+      def initialize username, password
+        check_gem
+        @username = username
+        @password = password
+      end
+      def check_gem # :nodoc:
+        require 'www/delicious'
+      rescue LoadError
+        warn_gem_missing 'www-delicious', 'the delicious source'
+        exit 1
       end
+
+      def to_s
+        "#{self.class.name}(#{@username})"
+      end
+
+      # Harvests the data to index.
+      #
+      def harvest category
+        get_data do |indexed_id, data|
+          text = data[category.from]
+          next unless text
+          yield indexed_id, text
+        end
+      end
+
+      #
+      #
+      def get_data # :nodoc:
+        @generated_id ||= 0
+        @posts ||= WWW::Delicious.new(@username, @password).posts_recent(count: 100)
+        @posts.each do |post|
+          data = {
+            title: post.title,
+            tags:  post.tags.join(' '),
+            url:   post.url.to_s
+          }
+          @generated_id += 1
+          yield @generated_id, data
+        end
+      end
+
     end
 
   end

data/lib/picky/sources/mongo.rb

@@ -1,75 +1,80 @@
-module Sources
+module Picky
 
-  # Raised when a Mongo source is instantiated without a valid uri.
-	#
-	# Important!
-  # You have to start your mongodb with --rest in order to use
-  # the rest / http interface
-	#
-  class NoMongoDBGiven < StandardError; end
+  module Sources
 
-  # Important note: We're not sure if this works already.
-  #
-  # A Mongo database source.
-  #
-  # Options:
-  # * url, db
-  # Example:
-  #   Sources::Mongo.new(:collection1, :collection2, :url => 'localhost:28017', :db => 'testdatabase')
-	# Be sure to escape the URL properly, e.g. # => %23 in the databasename if needed
-	#
-  # and all the options of a <tt>RestClient::Resource</tt>.
-  # See http://github.com/archiloque/rest-client.
-  #
-  class Mongo < Base
-		@@id_key = '_id'
+    # Important note: We're not sure if this works already.
     #
+    # A Mongo database source.
     #
-    def initialize *category_names, options
-      check_gem
+    # Options:
+    # * url, db
+    # Example:
+    #   Sources::Mongo.new(:collection1, :collection2, :url => 'localhost:28017', :db => 'testdatabase')
+  	# Be sure to escape the URL properly, e.g. # => %23 in the databasename if needed
+  	#
+    # and all the options of a <tt>RestClient::Resource</tt>.
+    # See http://github.com/archiloque/rest-client.
+    #
+    class Mongo < Base
 
-			unless options[:url] && options[:db]
-				raise_no_db_given(category_names)
-			end
+      # Raised when a Mongo source is instantiated without a valid uri.
+    	#
+    	# Important!
+      # You have to start your mongodb with --rest in order to use
+      # the rest / http interface
+    	#
+      class NoDBGiven < StandardError; end
 
-		  @db         = RestClient::Resource.new options.delete(:url), options
-			@database   = options.delete(:db)
-		  @key_format = options[:key_format] && options[:key_format].to_sym || :to_sym
-		end
+  		@@id_key = '_id'
+      #
+      #
+      def initialize *category_names, options
+        check_gem
 
-    # Tries to require the rest_client gem.
-    #
-    def check_gem # :nodoc:
-      require 'rest_client'
-    rescue LoadError
-      warn_gem_missing 'rest-client', 'the MongoDB source'
-      exit 1
-    end
+  			unless options[:url] && options[:db]
+  				raise_no_db_given(category_names)
+  			end
 
-		# Fetches the data, @limit=0 will return all records
-		#
-		# Limit is set to 0 by default - all collection entries will be send
-		# If want to limit the results, set to to any other number, e.g. limit=15
-		# to return only 15 entries
-		#
-		def harvest category
-			collection = (category.from || category.index_name).to_s
-			resp = @db["/#{@database}/#{category.index_name}/?@limit=0"].get
-			JSON.parse(resp)['rows'].each do |row|
-			  text = row[collection].to_s
-			  next unless text
-				index_key = row.delete(@@id_key) # TODO Still works, I removed .values
-				yield index_key, text
-			end
-		end
+  		  @db         = RestClient::Resource.new options.delete(:url), options
+  			@database   = options.delete(:db)
+  		  @key_format = options[:key_format] && options[:key_format].to_sym || :to_sym
+  		end
 
-    def raise_no_db_given category_names # :nodoc:
-      raise NoMongoDBGiven.new(category_names.join(', '))
-    end
+      # Tries to require the rest_client gem.
+      #
+      def check_gem # :nodoc:
+        require 'rest_client'
+      rescue LoadError
+        warn_gem_missing 'rest-client', 'the MongoDB source'
+        exit 1
+      end
 
-    def to_s
-      self.class.name
-    end
+  		# Fetches the data, @limit=0 will return all records
+  		#
+  		# Limit is set to 0 by default - all collection entries will be send
+  		# If want to limit the results, set to to any other number, e.g. limit=15
+  		# to return only 15 entries
+  		#
+  		def harvest category
+  			collection = (category.from || category.index_name).to_s
+  			resp = @db["/#{@database}/#{category.index_name}/?@limit=0"].get
+  			JSON.parse(resp)['rows'].each do |row|
+  			  text = row[collection].to_s
+  			  next unless text
+  				index_key = row.delete(@@id_key) # TODO Still works, I removed .values
+  				yield index_key, text
+  			end
+  		end
 
+      def raise_no_db_given category_names # :nodoc:
+        raise NoDBGiven.new(category_names.join(', '))
+      end
+
+      def to_s
+        self.class.name
+      end
+
+    end
   end
-end
+
+end

data/lib/picky/sources/wrappers/base.rb

@@ -1,24 +1,28 @@
-module Sources
+module Picky
 
-  # Source wrappers can be used to rewrite data before it goes into the index.
-  #
-  # For example if you want to normalize data.
-  #
-  module Wrappers # :nodoc:all
+  module Sources
 
-    class Base
+    # Source wrappers can be used to rewrite data before it goes into the index.
+    #
+    # For example if you want to normalize data.
+    #
+    module Wrappers # :nodoc:all
 
-      attr_reader :source
+      class Base
 
-      # Wraps an indexing category.
-      #
-      def initialize source
-        @source = source
-      end
+        attr_reader :source
+
+        # Wraps an indexing category.
+        #
+        def initialize source
+          @source = source
+        end
 
-      # Default is delegation for all methods
-      #
-      delegate :harvest, :connect_backend, :take_snapshot, :key_format, :to => :source
+        # Default is delegation for all methods
+        #
+        delegate :harvest, :connect_backend, :take_snapshot, :key_format, :to => :source
+
+      end
 
     end