picky 4.0.0pre1 → 4.0.0pre2

data/lib/picky/interfaces/live_parameters.rb

@@ -1,187 +0,0 @@
-module Picky
-
-  # This is very optional.
-  # Only load if the user wants it.
-  #
-  module Interfaces
-
-    # This is an interface that provides the user of
-    # Picky with the possibility to change parameters
-    # while the Application is running.
-    #
-    # Important Note: This will only work in Master/Child configurations.
-    #
-    class LiveParameters
-
-      def initialize
-        @child, @parent = IO.pipe
-        start_master_process_thread
-      end
-
-      # This runs a thread that listens to child processes.
-      #
-      def start_master_process_thread
-        # This thread is stopped in the children.
-        #
-        Thread.new do
-          loop do
-            IO.select([@child], nil, nil, 2) or next
-            result = @child.gets ';;;'
-            pid, configuration_hash = eval result
-            next unless Hash === configuration_hash
-            next if configuration_hash.empty?
-            exclaim "Trying to update MASTER configuration."
-            try_updating_configuration_with configuration_hash
-            kill_each_worker_except pid
-
-            # Fails hard on an error.
-            #
-          end
-        end
-      end
-
-      # TODO This needs to be webserver agnostic.
-      #
-      def worker_pids
-        Unicorn::HttpServer::WORKERS.keys
-      end
-
-      # Taken from Unicorn.
-      #
-      def kill_each_worker_except pid
-        worker_pids.each do |wpid|
-          next if wpid == pid
-          kill_worker :KILL, wpid
-        end
-      end
-      def kill_worker signal, wpid
-        Process.kill signal, wpid
-        exclaim "Killing worker ##{wpid} with signal #{signal}."
-      rescue Errno::ESRCH
-        remove_worker wpid
-      end
-      # TODO This needs to be Webserver agnostic.
-      #
-      def remove_worker wpid
-        worker = Unicorn::HttpServer::WORKERS.delete(wpid) and worker.tmp.close rescue nil
-      end
-
-      # Updates any parameters with the ones given and
-      # returns the updated params.
-      #
-      # The params are a strictly defined hash of:
-      #   * querying_removes_characters: Regexp
-      #   * querying_stopwords:          Regexp
-      #   * querying_splits_text_on:     Regexp
-      #
-      # This first tries to update in the child process,
-      # and if successful, in the parent process
-      #
-      def parameters configuration_hash
-        close_child
-        exclaim "Trying to update worker child configuration." unless configuration_hash.empty?
-        try_updating_configuration_with configuration_hash
-        write_parent configuration_hash
-        extract_configuration
-      rescue CouldNotUpdateConfigurationError => e
-        # I need to die such that my broken config is never used.
-        #
-        exclaim "Child process #{Process.pid} performs harakiri because of broken config."
-        harakiri
-        { e.config_key => :ERROR }
-      end
-      # Kills itself, but still answering the request honorably.
-      #
-      def harakiri
-        Process.kill :QUIT, Process.pid
-      end
-      # Write the parent.
-      #
-      # Note: The ;;; is the end marker for the message.
-      #
-      def write_parent configuration_hash
-        @parent.write "#{[Process.pid, configuration_hash]};;;"
-      end
-      # Close the child if it isn't yet closed.
-      #
-      def close_child
-        @child.close unless @child.closed?
-      end
-
-      class CouldNotUpdateConfigurationError < StandardError
-        attr_reader :config_key
-        def initialize config_key, message
-          super message
-          @config_key = config_key
-        end
-      end
-
-      # Tries updating the configuration in the child process or parent process.
-      #
-      def try_updating_configuration_with configuration_hash
-        current_key = nil
-        begin
-          configuration_hash.each_pair do |key, new_value|
-            exclaim "  Setting #{key} with #{new_value}."
-            current_key = key
-            send :"#{key}=", new_value
-          end
-        rescue StandardError => e
-          # Catch any error and reraise as config error.
-          #
-          raise CouldNotUpdateConfigurationError.new current_key, e.message
-        end
-      end
-
-      def extract_configuration
-        {
-          querying_removes_characters: querying_removes_characters,
-          querying_stopwords:          querying_stopwords,
-          querying_splits_text_on:     querying_splits_text_on
-        }
-      end
-
-      # TODO Move to Interface object.
-      #
-      def querying_removes_characters
-        regexp = Tokenizer.query_default.instance_variable_get :@removes_characters_regexp
-        regexp && regexp.source
-      end
-      def querying_removes_characters= new_value
-        Tokenizer.query_default.instance_variable_set(:@removes_characters_regexp, %r{#{new_value}})
-      end
-      def querying_stopwords
-        regexp = Tokenizer.query_default.instance_variable_get :@remove_stopwords_regexp
-        regexp && regexp.source
-      end
-      def querying_stopwords= new_value
-        Tokenizer.query_default.instance_variable_set(:@remove_stopwords_regexp, %r{#{new_value}})
-      end
-      def querying_splits_text_on
-        splits = Tokenizer.query_default.instance_variable_get :@splits_text_on
-        splits && splits.respond_to?(:source) ? splits.source : splits
-      end
-      def querying_splits_text_on= new_value
-        splits = Tokenizer.query_default.instance_variable_get :@splits_text_on
-        if splits.respond_to?(:source)
-          Tokenizer.query_default.instance_variable_set(:@splits_text_on, %r{#{new_value}})
-        else
-          Tokenizer.query_default.instance_variable_set(:@splits_text_on, new_value)
-        end
-      end
-
-      #
-      #
-      def to_s
-        "Suckerfish Live Interface (Use the picky-live gem to introspect)"
-      end
-
-    end
-
-  end
-
-  # Aka.
-  #
-  LiveParameters = Interfaces::LiveParameters
-
-end

data/lib/picky/sources/base.rb

@@ -1,92 +0,0 @@
-module Picky
-
-  # = Data Sources
-  #
-  # Currently, Picky offers the following Sources:
-  # * CSV (comma – or other – separated file)
-  # * Couch (CouchDB, key-value store)
-  # * DB (Databases, foremost MySQL)
-  # * Delicious (http://del.icio.us, online bookmarking service)
-  # See also:
-  # http://github.com/floere/picky/wiki/Sources-Configuration
-  #
-  # Don't worry if your source isn't here. Adding your own is easy:
-  # http://github.com/floere/picky/wiki/Contributing-sources
-  #
-  module Sources
-
-    # Sources are where your data comes from.
-    #
-    # A source has 1 mandatory and 2 optional methods:
-    # * connect_backend (_optional_): called once for each type/category pair.
-    # * harvest: Used by the indexer to gather data. Yields an indexed_id (string or integer) and a string value.
-    # * take_snapshot (_optional_): called once for each index or category (if indexing a single category).
-    #
-    # This base class "implements" all these methods, but they don't do anything.
-    # Subclass this class <tt>class MySource < Base</tt> and override the methods in your source to do something.
-    #
-    class Base
-
-      attr_reader :key_format
-
-      # Connect to the backend.
-      #
-      # Called once per index/category combination before harvesting.
-      #
-      # Examples:
-      # * The DB backend connects the DB adapter.
-      # * We open a connection to a key value store.
-      # * We open an file with data.
-      #
-      def connect_backend
-
-      end
-
-      # Called by the indexer when gathering data.
-      #
-      # Yields the data (id, text for id) for the given category.
-      #
-      # When implementing or overriding your own,
-      # be sure to <tt>yield(id, text_for_id)</tt> (or <tt>block.call(id, text_for_id)</tt>)
-      # for the given type symbol and category symbol.
-      #
-      # Note: Since harvest needs to be implemented, it has no default impementation.
-      #
-      # def harvest category # :yields: id, text_for_id
-      #
-      # end
-
-      # Used to take a snapshot of your data if it is fast changing.
-      #
-      # Called once for each index before harvesting.
-      # If it has been called on a source already by an index,
-      # it won't be called again for a category inside that index.
-      #
-      # Example:
-      # * In a DB source, a table based on the source's select statement is created.
-      #
-      def take_snapshot index
-
-      end
-
-      # Used to check if a snapshot has been done already.
-      #
-      # Example:
-      # * In a DB source, a table based on the source's select statement is created.
-      #
-      def with_snapshot index
-        connect_backend
-        @snapshot_taken ||= 0
-        if @snapshot_taken.zero?
-          take_snapshot index
-        end
-        @snapshot_taken += 1
-        yield
-        @snapshot_taken -= 1
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/sources/couch.rb

@@ -1,76 +0,0 @@
-module Picky
-
-  module Sources
-
-    # A Couch database source.
-    #
-    # Options:
-    # * url
-    # and all the options of a <tt>RestClient::Resource</tt>.
-    # See http://github.com/archiloque/rest-client.
-    #
-    # Examples:
-    #  Picky::Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984')
-    #  Picky::Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984', user:'someuser', password:'somepassword')
-    #
-    class Couch < Base
-
-      # Raised when a Couch source is instantiated without a file.
-      #
-      # Example:
-      #   Picky::Sources::Couch.new(:column1, :column2) # without file option
-      #
-      class NoDBGiven < StandardError; end
-
-      #
-      #
-      def initialize *category_names, options
-        check_gem
-
-        Hash === options && options[:url] || raise_no_db_given(category_names)
-
-        @db = RestClient::Resource.new options.delete(:url), options
-
-        key_format   = options.delete :key_format
-        @key_format  = key_format && key_format.intern || :to_s
-      end
-
-      def to_s
-        self.class.name
-      end
-
-      # Tries to require the rest_client gem.
-      #
-      def check_gem # :nodoc:
-        require 'rest_client'
-      rescue LoadError
-        warn_gem_missing 'rest-client', 'the CouchDB source'
-        exit 1
-      end
-
-      # Harvests the data to index.
-      #
-      # See important note, above.
-      #
-      @@id_key = '_id'
-      def harvest category
-        category_name = category.from.to_s
-        get_data do |doc|
-          yield doc[@@id_key], doc[category_name] || next
-        end
-      end
-
-      def get_data &block # :nodoc:
-        resp = @db['_all_docs?include_docs=true'].get
-        JSON.parse(resp)['rows'].
-          map{|row| row['doc']}.
-          each &block
-      end
-
-      def raise_no_db_given category_names # :nodoc:
-        raise NoDBGiven.new(category_names.join(', '))
-      end
-    end
-  end
-
-end

data/lib/picky/sources/csv.rb

@@ -1,83 +0,0 @@
-module Picky
-
-  module Sources
-
-    # Describes a CSV source, a file with comma separated values in it.
-    #
-    # The first column is implicitly assumed to be the id column.
-    #
-    # It takes the same options as the Ruby 1.9 CSV class.
-    #
-    # Examples:
-    #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv')
-    #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', col_sep:';')
-    #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', row_sep:"\n")
-    #
-    class CSV < Base
-
-      # Raised when a CSV source is instantiated without a file.
-      #
-      # Example:
-      #   Sources::CSV.new(:column1, :column2) # without file option
-      #
-      class NoFileGiven < StandardError; end
-
-      # The CSV file's path, relative to PICKY_ROOT.
-      #
-      attr_reader :file_name
-
-      # The options that were passed into #new.
-      #
-      attr_reader :csv_options, :key_format
-
-      # The data category names.
-      #
-      attr_reader :category_names
-
-      def initialize *category_names, options
-        require 'csv'
-        @category_names = category_names
-
-        @csv_options = Hash === options && options || {}
-        @file_name   = @csv_options.delete(:file) || raise_no_file_given(category_names)
-
-        key_format   = options.delete :key_format
-        @key_format  = key_format && key_format.intern || :to_i
-      end
-
-      def to_s
-        parameters = category_names
-        parameters << { file: file_name }
-        parameters << csv_options unless csv_options.empty?
-        %Q{#{self.class.name}(#{parameters.join(', ')})}
-      end
-
-      # Raises a NoCSVFileGiven exception.
-      #
-      def raise_no_file_given category_names # :nodoc:
-        raise NoFileGiven.new(category_names.join(', '))
-      end
-
-      # Harvests the data to index.
-      #
-      def harvest category
-        index = category_names.index category.from
-        get_data do |indexed_id, *ary|
-          text = ary[index]
-          next unless text
-          text.force_encoding 'utf-8' # TODO Still needed?
-          yield indexed_id, text
-        end
-      end
-
-      #
-      #
-      def get_data &block # :nodoc:
-        ::CSV.foreach file_name, csv_options, &block
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/sources/db.rb

@@ -1,189 +0,0 @@
-module Picky
-
-  module Sources
-
-    # Describes a database source. Needs a SELECT statement
-    # (with id in it), and a file option or the options from an AR config file.
-    #
-    # The select statement can be as complicated as you want,
-    # as long as it has an id in it and as long as it can be
-    # used in a CREATE TABLE AS statement.
-    # (working on that last one)
-    #
-    # Examples:
-    #  Sources::DB.new('SELECT id, title, author, year FROM books') # Uses the config from app/db.yml by default.
-    #  Sources::DB.new('SELECT id, title, author, year FROM books', file: 'app/some_db.yml')
-    #  Sources::DB.new('SELECT b.id, b.title, b.author, b.publishing_year as year FROM books b INNER JOIN ON ...', file: 'app/some_db.yml')
-    #  Sources::DB.new('SELECT id, title, author, year FROM books', adapter: 'mysql', host:'localhost', ...)
-    #
-    class DB < Base
-
-      # The select statement that was passed in.
-      #
-      attr_reader :select_statement
-
-      # The database adapter.
-      #
-      attr_reader :database
-
-      # The database connection options that were either passed in or loaded from the given file.
-      #
-      attr_reader :connection_options, :options
-
-      @@traversal_id = :__picky_id
-
-      def initialize select_statement, options = { file: 'app/db.yml' }
-        check_gem
-
-        @select_statement = select_statement
-        @database         = create_database_adapter
-        @options          = options
-      end
-
-      # Tries to require the active_record gem.
-      #
-      def check_gem # :nodoc:
-        require 'active_record'
-      rescue LoadError
-        warn_gem_missing 'active_record', 'the (ActiveRecord) DB source'
-        exit 1
-      end
-
-      def to_s
-        parameters = [select_statement.inspect]
-        parameters << options unless options.empty?
-        %Q{#{self.class.name}(#{parameters.join(', ')})}
-      end
-
-      # Creates a database adapter for use with this source.
-      #
-      def create_database_adapter # :nodoc:
-        # THINK Do not use ActiveRecord directly? Use set_table_name etc?
-        #
-        adapter_class = Class.new ActiveRecord::Base
-        adapter_class.abstract_class = true
-        adapter_class
-      end
-
-      # Configure the backend.
-      #
-      # Options:
-      #  Either
-      #  * file => 'some/filename.yml' # With an active record configuration.
-      #  Or
-      #  * The configuration as a hash.
-      #
-      def configure options # :nodoc:
-        @connection_options = if filename = options[:file]
-          File.open(File.join(PICKY_ROOT, filename)) { |file| YAML::load(file) }
-        else
-          options
-        end
-        self
-      end
-
-      # Connect the backend.
-      #
-      # Will raise unless connection options have been given.
-      #
-      def connect_backend
-        configure @options
-        raise "Database backend not configured" unless connection_options
-        database.establish_connection connection_options
-      end
-
-      # Take a snapshot of the data.
-      #
-      # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
-      #
-      def take_snapshot index
-        timed_exclaim %Q{  "#{index.identifier}": Taking snapshot of database data.}
-
-        origin = snapshot_table_name index.name
-        on_database = database.connection
-
-        # Drop the table if it exists.
-        #
-        on_database.drop_table origin if on_database.table_exists?(origin)
-
-        # The adapters currently do not support this.
-        #
-        on_database.execute "CREATE TABLE #{origin} AS #{select_statement}"
-
-        # Add a column that Picky uses to traverse the table's entries.
-        #
-        on_database.add_column origin, @@traversal_id, :primary_key, :null => :false
-
-        # Execute any special queries this index needs executed.
-        #
-        on_database.execute index.after_indexing if index.after_indexing
-      end
-
-      # Counts all the entries that are used for the index.
-      #
-      def count index_name
-        database.connection.select_value("SELECT COUNT(#{@@traversal_id}) FROM #{snapshot_table_name(index_name)}").to_i
-      end
-
-      # The name of the snapshot table created by Picky.
-      #
-      def snapshot_table_name index_name
-        "picky_#{index_name}_index"
-      end
-
-      # Harvests the data to index in chunks.
-      #
-      def harvest category, &block
-        (0..count(category.index_name)).step(chunksize) do |offset|
-          get_data category, offset, &block
-        end
-      end
-
-      # Gets the data from the backend.
-      #
-      def get_data category, offset, &block # :nodoc:
-        select_statement = harvest_statement_with_offset category, offset
-
-        # THINK Not really nice like this. Rewrite if possible.
-        #
-        if database.connection.adapter_name == "PostgreSQL"
-          id_key   = 'id'
-          text_key = category.from.to_s
-          database.connection.execute(select_statement).each do |hash|
-            id, text = hash.values_at id_key, text_key
-            yield id, text if text
-          end
-        else
-          database.connection.execute(select_statement).each do |id, text|
-            yield id, text if text
-          end
-        end
-      end
-
-      # Builds a harvest statement for getting data to index.
-      #
-      def harvest_statement_with_offset category, offset
-        statement = harvest_statement category
-
-        statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
-
-        "#{statement} st.#{@@traversal_id} > #{offset} LIMIT #{chunksize}"
-      end
-
-      # The harvest statement used to pull data from the snapshot table.
-      #
-      def harvest_statement category
-        "SELECT id, #{category.from} FROM #{snapshot_table_name(category.index_name)} st"
-      end
-
-      # The amount of records that are loaded each chunk.
-      #
-      def chunksize
-        25_000
-      end
-
-    end
-
-  end
-
-end