search_steroids 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= SearchTerms
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,39 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'SearchTerms'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+# include the dummy app rake tasks, so that you can create a test db etc from the root.
+import 'spec/dummy/Rakefile'
+task :default => :test

data/lib/generators/enable_tsearch/USAGE

@@ -0,0 +1,14 @@
+    rails generate enable_tsearch <table> <context> <col1> <col2> ...
+
+This assumes that you have already created a search_context using 'rails g search_context ...' and want to enable using that search_context from this related model.
+
+This would generate the following
+
+1. migration to add a context_vector column to table
+2. migration to create a trigger that updates the context_vector column from <col1> <col2> ... whenever <table> is inserted or updated.
+
+The migration to create the trigger is only an example, it'll work for simple cases but you'd need to edit it to create more complex and useful triggers, for example
+if you needed to join to a remote table to get some columns from there.
+
+
+  

data/lib/generators/enable_tsearch/enable_tsearch_generator.rb

@@ -0,0 +1,45 @@
+require 'rails/generators'
+require 'rails/generators/active_model'
+require 'rails/generators/migration'
+require 'generators/generator_helpers.rb'
+class EnableTsearchGenerator < Rails::Generators::Base  
+  include Rails::Generators::Migration
+  source_root File.expand_path('../templates', __FILE__)  
+  argument :table_name, :type => :string
+  argument :search_context, :type=>:string
+  argument :columns, :type => :array, :banner => "col1 col2 col3 ..."
+   
+  
+  def self.next_migration_number(path)
+      sleep(1) # force a new timestamp
+      Time.now.utc.strftime("%Y%m%d%H%M%S")
+  end
+    
+  def create_context
+    tmp = "  search_context [:#{columns.join(',:')}], :search_context=>:#{search_context}\n" unless columns.empty?
+    tmp = "  search_context :a_method_that_returns_the_search_terms, :search_context=>:#{search_context}\n" if columns.empty?
+    inject_into_file "app/models/#{table_name.singularize}.rb", tmp , :after => /class #{table_name.singularize.camelize} < .*\n/
+    migrate_if_needed("add_column_migration.rb","db/migrate/add_#{column_name}_to_#{table_name}.rb")
+    migrate_if_needed("add_trigger_migration.rb", "db/migrate/add_#{search_context}_trigger_to_#{table_name}.rb")
+    migrate_if_needed("add_tsearch_index_migration.rb", "db/migrate/add_#{search_context}_tsearch_index_to_#{table_name}.rb")
+    migrate_if_needed("populate_vector_migration.rb", "db/migrate/populate_#{column_name}_on_#{table_name}.rb")
+    migrate_if_needed("populate_search_terms_migration.rb", "db/migrate/populate_#{search_context}_from_#{table_name}.rb")
+  end
+  
+  protected
+  def column_name
+    "#{search_context}_vector"
+  end
+  def search_config_name
+     "#{search_context}_search_config"
+   end
+  def index_name
+    "idx_#{column_name}_on_#{table_name}"    
+  end
+  def trigger_name
+    "update_#{column_name}"    
+  end
+  def trigger_sp_name
+    "sp_update_#{column_name}"    
+  end
+end

data/lib/generators/enable_tsearch/templates/add_column_migration.rb

@@ -0,0 +1,5 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def change
+    add_column :<%=table_name %>, :<%=column_name %>, :tsvector
+  end
+end

data/lib/generators/enable_tsearch/templates/add_trigger_migration.rb

@@ -0,0 +1,27 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up
+    execute <<-EOS
+    -- feel free to customize this function if you want to join to columns from another table, for example
+    CREATE OR REPLACE FUNCTION <%=trigger_sp_name %>() RETURNS trigger AS $$
+    begin
+      new.<%=column_name %> :=
+      <% sep='';columns.each do |column| -%>
+            <%=sep%>setweight(to_tsvector('<%=search_config_name %>', coalesce(new.<%=column %>,'')), 'A')
+            <%sep='|| ' -%>
+      <% end -%>;
+      return new;
+    end
+    $$ LANGUAGE plpgsql;
+    EOS
+    execute <<-EOS
+    DROP TRIGGER IF EXISTS <%=trigger_name %> ON <%=table_name %>;
+    CREATE TRIGGER <%=trigger_name %> BEFORE INSERT OR UPDATE
+        ON <%=table_name %> FOR EACH ROW EXECUTE PROCEDURE <%=trigger_sp_name %>();
+    EOS
+  end
+  def down
+    execute "DROP TRIGGER <%=trigger_name %>"
+    execute "DROP FUNCTION <%=trigger_sp_name %>"
+    
+  end
+end

data/lib/generators/enable_tsearch/templates/add_tsearch_index_migration.rb

@@ -0,0 +1,8 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up 
+    execute "CREATE INDEX <%=index_name %> on <%=table_name %> USING gin (<%=column_name%>)"
+  end
+  def down
+    execute "DROP INDEX <%=index_name %>"
+  end
+end

data/lib/generators/enable_tsearch/templates/populate_search_terms_migration.rb

@@ -0,0 +1,21 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up
+      execute <<-EOS
+       -- DELETE from <%=search_context %>;
+       INSERT into <%=search_context %>(name, count,created_at, updated_at)
+       SELECT word, ndoc as count, now() as created_at, now() as updated_at
+       FROM ts_stat(
+       'SELECT to_tsvector(''simple'',
+       <% sep='';columns.each do |column| -%>
+             <%=sep%>coalesce(<%=column %>,'''')
+             <%sep=" || '' '' ||" -%>
+       <% end -%>
+       ) FROM <%=table_name %>
+       ')
+      EOS
+  end
+  def down
+    execute %Q{DELETE from <%=search_context %>}
+  end
+end
+

data/lib/generators/enable_tsearch/templates/populate_vector_migration.rb

@@ -0,0 +1,8 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up
+    # nop update that hopefully triggers the TRIGGER to update the vector
+    execute %Q{UPDATE <%=table_name%> set updated_at = updated_at}
+  end
+  def down
+  end
+end

data/lib/generators/generator_helpers.rb

@@ -0,0 +1,10 @@
+module SearchContext
+  module GeneratorHelpers  
+    def migrate_if_needed(template,migration)
+      unless self.class.migration_exists?(File.dirname(migration),File.basename(migration).gsub('.rb',''))
+        migration_template template, migration
+      end   
+    end
+  end
+end
+Rails::Generators::Base.send :include, SearchContext::GeneratorHelpers

data/lib/generators/search_context/USAGE

@@ -0,0 +1,8 @@
+    rails generate search_context <context>
+
+The <context> is the both the context of the words (ie, names) and the table that is used to store them in the database. Typically you only need one context, and the default name is 'search_terms', so you would type: 
+
+    rails g search_context
+    
+That will generate the search_terms migration + model.
+  

data/lib/generators/search_context/search_context_generator.rb

@@ -0,0 +1,79 @@
+require 'rails/generators'
+require 'rails/generators/active_model'
+require 'rails/generators/migration'
+require 'generators/generator_helpers.rb'
+
+class SearchContextGenerator < Rails::Generators::Base  
+  include Rails::Generators::Migration
+  source_root File.expand_path('../templates', __FILE__)  
+  argument :context, :type => :string, :banner => "<context-name>"
+  class_option :dynamic, :type => :boolean, :default => false, :description => "include count so that the search context can be maintained dynamically"
+   
+  
+  def self.next_migration_number(path)
+      sleep(1) # force a new timestamp
+      Time.now.utc.strftime("%Y%m%d%H%M%S")
+  end
+    
+  def create_context
+    migrate_if_needed "install_migration.rb", "db/migrate/install_trigram_extension.rb"
+    template 'model.rb', "app/models/#{model_file_name}.rb"
+    template 'alias.rb', "app/models/#{aliases_file_name}.rb"
+    template 'migration_helper.rb', "db/migration_helper.rb"
+    migrate_if_needed "create_model_migration.rb", "db/migrate/create_#{table_name}.rb"
+    migrate_if_needed "create_search_config_migration.rb", "db/migrate/add_#{search_config_name}.rb"
+    migrate_if_needed "add_context_migration.rb", "db/migrate/add_trigram_index_to_#{context}.rb"
+    migrate_if_needed "create_aliases_migration.rb", "db/migrate/create_#{aliases_name}.rb"
+    migrate_if_needed "add_aliases_trigger_migration.rb", "db/migrate/add_#{aliases_name}_trigger.rb"
+  end
+  
+  protected
+
+  def column_name
+    "#{context}_vector"
+  end
+
+  def class_name
+    model_file_name.camelize
+  end
+
+  def model_file_name
+    context.underscore.singularize
+  end
+
+  def search_config_name
+    "#{table_name}_search_config"
+  end
+
+  def alias_search_config_name
+    "#{table_name}_alias_search_config"
+  end
+  
+  def aliases_file_name
+    "#{table_name.singularize}_alias"
+  end
+
+  def aliases_name
+    "#{table_name.singularize}_aliases"
+  end
+
+  def aliases_class_name
+     "#{aliases_name.singularize.camelize}"
+   end
+
+  def table_name
+    context.underscore.pluralize
+  end
+  def index_name
+    "idx_trgm_on_#{table_name}"    
+  end
+  
+  def trigger_name
+    "trigger_update_#{aliases_name}"    
+  end
+
+  def trigger_sp_name
+    "sp_update_#{aliases_name}"    
+  end
+  
+end

data/lib/generators/search_context/templates/add_aliases_trigger_migration.rb

@@ -0,0 +1,23 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up
+    execute <<-EOS
+    CREATE OR REPLACE FUNCTION <%=trigger_sp_name %>() RETURNS trigger AS $$
+    begin
+      new.original_tsquery = plainto_tsquery('<%=alias_search_config_name%>', coalesce(new.original,''));
+      new.substitution_tsquery = plainto_tsquery('<%=alias_search_config_name%>', coalesce(new.substitution,''));
+      return new;
+    end
+    $$ LANGUAGE plpgsql;
+    EOS
+    execute <<-EOS
+    DROP TRIGGER IF EXISTS <%=trigger_name %> ON <%=aliases_name %>;
+    CREATE TRIGGER <%=trigger_name %> BEFORE INSERT OR UPDATE
+        ON <%=aliases_name %> FOR EACH ROW EXECUTE PROCEDURE <%=trigger_sp_name %>();
+    EOS
+  end
+  
+  def down
+    execute "DROP TRIGGER <%=trigger_name %> on <%=aliases_name %>;"
+    execute "DROP FUNCTION <%=trigger_sp_name(); %>"    
+  end
+end

data/lib/generators/search_context/templates/add_context_migration.rb

@@ -0,0 +1,8 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up 
+    execute "CREATE INDEX <%=index_name %> on <%=table_name %> USING gin (name gin_trgm_ops)"
+  end
+  def down
+    execute "DROP INDEX <%=index_name %>"
+  end
+end

data/lib/generators/search_context/templates/alias.rb

@@ -0,0 +1,4 @@
+class <%=aliases_class_name %> < ActiveRecord::Base
+  # database trigger maintains *_tsquery versions
+  attr_accessible :original, :substitution
+end

data/lib/generators/search_context/templates/create_aliases_migration.rb

@@ -0,0 +1,10 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def change
+    # populate this table with your query rewrite rules
+    create_table :<%=aliases_name %> do |t| t.timestamps end
+    add_column :<%=aliases_name %>, :original_tsquery, :tsquery
+    add_column :<%=aliases_name %>, :substitution_tsquery, :tsquery
+    add_column :<%=aliases_name %>, :original, :string
+    add_column :<%=aliases_name %>, :substitution, :string
+  end
+end

data/lib/generators/search_context/templates/create_model_migration.rb

@@ -0,0 +1,11 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def change
+    create_table :<%=table_name %> do |t|
+      t.string :name
+      <%if options.dynamic? -%>
+        t.integer :count
+      <%end%>
+      t.timestamps
+    end
+  end
+end

data/lib/generators/search_context/templates/create_search_config_migration.rb

@@ -0,0 +1,41 @@
+require File.expand_path('../../migration_helper', __FILE__)
+include MigrationHelper
+
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up
+     # install unaccent if it isn't installed'
+     install_extension('unaccent')
+
+     # create a new search configuration by copying an existing one. For my purposes, I wanted to use the french dictionary and
+     # remove all the accent characters. You could equally use the english dictionary as your base, or create your own.
+     # this is only intended to show you the basic command.
+     
+     execute %Q{CREATE TEXT SEARCH CONFIGURATION <%=search_config_name%> ( COPY = pg_catalog.french );}
+    # From the docs, you typically don't want these mappings for simple searches of names and names of things, so I show you how to 
+    # drop them here
+    #   Parsing documents into tokens. It is useful to identify various classes of tokens, e.g., numbers, words, complex words,
+    #   email addresses, so that they can be processed differently.
+    #   In principle token classes depend on the specific application, but for most purposes it 
+    #   is adequate to use a predefined set of classes. PostgreSQL uses a parser to perform this step.
+    #    A standard parser is provided, and custom parsers can be created for specific needs.
+     execute %{ALTER TEXT SEARCH CONFIGURATION <%=search_config_name%>
+            DROP MAPPING FOR email, file, float, host, int, sfloat, uint, url, url_path, version}
+    # remove all the accent characters. 
+     execute %Q{ALTER TEXT SEARCH CONFIGURATION <%=search_config_name%>
+             ALTER MAPPING FOR hword, hword_part, word
+             WITH unaccent, french_stem;}
+    
+    # create a variant of the simple dictionary that ignores accents
+    execute %Q{CREATE TEXT SEARCH CONFIGURATION <%=alias_search_config_name%> ( COPY = pg_catalog.simple );}
+    execute %{ALTER TEXT SEARCH CONFIGURATION <%=alias_search_config_name%>
+           DROP MAPPING FOR email, file, float, host, int, sfloat, uint, url, url_path, version}
+    execute %Q{ALTER TEXT SEARCH CONFIGURATION <%=alias_search_config_name%>
+            ALTER MAPPING FOR hword, hword_part, word
+            WITH unaccent, simple;}
+  end
+
+  def down
+    execute "DROP TEXT SEARCH CONFIGURATION <%=search_config_name%>"
+    execute "DROP TEXT SEARCH CONFIGURATION <%=alias_search_config_name%>"
+  end
+end

data/lib/generators/search_context/templates/install_migration.rb

@@ -0,0 +1,12 @@
+require File.expand_path('../../migration_helper', __FILE__)
+include MigrationHelper
+
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def up
+    install_extension('pg_trgm')
+  end
+
+  def down
+    uninstall_extension('pg_trgm')
+  end
+end

data/lib/generators/search_context/templates/migration_helper.rb

@@ -0,0 +1,42 @@
+module MigrationHelper
+  def support_create_extension?
+    # Assume that 9.1 or later supports CREATE EXTENSION
+    version_parts = select_value("SELECT version()").match(/PostgreSQL ([\d\.]*)/)[1].split('.')
+    version_parts[0].to_i >= 9 && version_parts[1].to_i >= 1
+  end
+
+  # Install the extension named extension_name
+  def install_extension(extension_name, function_name = nil)
+    if support_create_extension?
+      if select_value(%Q{select extname from pg_extension where extname = '#{extension_name}'}).nil?
+        execute "CREATE EXTENSION #{extension_name}"
+      end
+    else
+      # Speculating that the extension installs a function of the same name
+      function_name ||= extension_name
+      unless select_value("SELECT proname FROM pg_proc WHERE proname = '#{function_name}'")
+         puts "*" * 60
+         puts ""
+         puts "You need to install the #{extension_name} extension"
+         puts ""
+         sharedir = `pg_config --sharedir 2>&1`.strip
+         if $?.to_i > 0
+           puts "First, you need to know SHAREDIR - maybe something like `pg_config --sharedir`?"
+           puts ""
+           sharedir = 'SHAREDIR'
+         end
+         puts "Try `psql -d #{connection.current_database} -f #{sharedir}/contrib/#{extension_name}.sql`"
+         puts ""
+         puts "*" * 60
+         raise "Extension #{name} not installed."
+      end
+    end
+  end
+  def uninstall_extension(extension_name)
+    if support_create_extension?
+      execute "DROP EXTENSION #{extension_name}"
+    else
+      puts "Skipping uninstall of extension #{extension_name}"
+    end
+  end
+end