calagator 0.0.1.pre1

data/db/seeds.rb

@@ -0,0 +1,16 @@
+# ruby encoding: utf-8
+
+# This file should contain all the record creation needed to seed the database with its default values.
+# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+#
+# Examples:
+#
+#   cities = City.create([{ :name => 'Chicago' }, { :name => 'Copenhagen' }])
+#   Mayor.create(:name => 'Daley', :city => cities.first)
+
+require 'faker'
+
+FactoryGirl.create_list(:seed_venue, 25, :with_events)
+FactoryGirl.create_list(:seed_venue, 25)
+FactoryGirl.create_list(:seed_event, 25, :with_venue)
+FactoryGirl.create_list(:seed_event, 25)

data/lib/calagator.rb

@@ -0,0 +1,30 @@
+require "calagator/engine"
+
+require "formtastic"
+require "rails_autolink"
+require "nokogiri"
+require "columnize"
+require "geokit"
+require "htmlentities"
+require "paper_trail"
+require "ri_cal"
+require "will_paginate"
+require "will_paginate/array"
+require "rest-client"
+require "loofah"
+require "loofah-activerecord"
+require "bluecloth"
+require "formtastic"
+require "acts-as-taggable-on"
+require "jquery-rails"
+require "jquery-ui-rails"
+require "progress_bar"
+require "font-awesome-rails"
+require "paper_trail_manager"
+require "utf8-cleaner"
+require "sunspot_rails"
+require "sunspot_solr"
+require "lucene_query"
+
+module Calagator
+end

data/lib/calagator/blacklist_validator.rb

@@ -0,0 +1,69 @@
+# = BlacklistValidator
+#
+# A naively simple mixin that blacklists content in ActiveModel objects.
+#
+# == Usage
+#
+# Let's say that your applications lets people post messages, but don't want
+# them using the word "viagra" in posts as a naively simple way of preventing
+# spam.
+#
+# You'd first create a config/blacklist.txt file with a line like:
+#   \bviagrab\b
+#
+# And then you'd include the blacklisting feature into your Message class like:
+#
+#   class Message < ActiveRecord::Base
+#     validates :title, :content, blacklist: true
+#   end
+#
+# Now including the word "viagra" in your record's values will fail:
+#
+#   message = Message.new(title: "foo viagra bar")
+#   message.valid? # => false
+#
+# Available validator options:
+#   * patterns: Array of regular expressions that will be matched
+#     against the given attribute contents and any matches will cause the
+#     record to be marked invalid.
+#   * blacklist: Reads an array of blacklisted regular expressions from
+#     a filename.
+#   * message: Error message to use on invalid records.
+#
+# If no :patterns or :blacklist is given, patterns are read from:
+#   * config/blacklist.txt
+#   * config/blacklist-local.txt
+
+class BlacklistValidator < ActiveModel::EachValidator
+  BLACKLIST_DEFAULT_MESSAGE = "contains blacklisted content"
+
+  def validate_each(record, attribute, value)
+    if value.present? && patterns.any? { |pattern| value.match(pattern) }
+      record.errors.add attribute, message
+    end
+  end
+
+  private
+
+  def message
+    options.fetch(:message, BLACKLIST_DEFAULT_MESSAGE)
+  end
+
+  def patterns
+    @patterns ||= options.fetch(:patterns) do
+      [
+        Calagator.blacklist_patterns,
+        get_blacklist_patterns_from(options.fetch(:blacklist, "blacklist.txt")),
+      ].flatten.compact
+    end
+  end
+
+  def get_blacklist_patterns_from(filename)
+    filename = Rails.root.join('config',filename) unless filename.match(/[\/\\]/)
+    return unless File.exists?(filename)
+
+    File.readlines(filename).map do |line|
+      Regexp.new(line.strip, Regexp::IGNORECASE)
+    end
+  end
+end

data/lib/calagator/decode_html_entities_hack.rb

@@ -0,0 +1,33 @@
+# = DecodeHtmlEntitiesHack
+#
+# Loofah encodes HTML entities in every string that gets passed through it.
+# This includes things like the friendly ampersand which we don't want to be HTML
+# encoded in our database, so we need to decode the things that it changes.
+#
+# This should be included in models that use xss_foliate somewhere _after_ the xss_folidate call.
+#
+# This is, as the name of the module suggests, a giant hack. At some point, it should
+# be removed after the issue is resolved in the underlying library:
+# https://github.com/flavorjones/loofah/issues/20#issuecomment-1751538
+#
+# Warning: this effectively renders loofah's "escape" scrubbing mode useless by
+# undoing everything it does. Don't use that mode.
+#
+module Calagator
+
+module DecodeHtmlEntitiesHack
+  def self.included(base)
+    base.set_callback(:validate, :before, :decode_html_entities)
+  end
+
+  def decode_html_entities
+    self.attributes.each do |field, value|
+      decoded_content = HTMLEntities.new.decode(value)
+      if decoded_content.present? && !(decoded_content == value)
+        self.send("#{field}=", decoded_content)
+      end
+    end
+  end
+end
+
+end

data/lib/calagator/duplicate_checking.rb

@@ -0,0 +1,133 @@
+# = DuplicateChecking
+#
+# This mixin provides a way for ActiveRecord classes to find and squash duplicates.
+#
+# Example:
+#
+#   # Define your class
+#   class Thing < ActiveRecord::Base
+#     # Load the mixin into your class
+#     include DuplicateChecking
+#
+#     # Declare attributes that should be ignored during duplicate checks
+#     duplicate_checking_ignores_attributes :random_value
+#
+#     # Declare associations that should be ignored during duplicate squashing
+#     duplicate_squashing_ignores_associations :tags
+#   end
+#
+#   # Set duplicates on objects
+#   foo1 = Thing.create!!:name => "foo")
+#   foo2 = Thing.create!(:name => "foo", :duplicate_of => foo1)
+#   bar  = Thing.create!(:name => "bar")
+#
+#   # Check whether record is set as duplicate
+#   foo1.duplicate? # => false
+#   foo2.duplicate? # => true
+#   bar.duplicate?  # => false
+#
+#   # Find duplicate of a record
+#   foo3.find_exact_duplicates # => [foo1, foo2]
+#   bar.find_exact_duplicates  # => nil
+require "calagator/duplicate_checking/duplicate_finder"
+require "calagator/duplicate_checking/duplicate_squasher"
+
+module Calagator
+
+module DuplicateChecking
+  DUPLICATE_MARK_COLUMN = :duplicate_of_id
+  DEFAULT_SQUASH_METHOD = :mark
+  DUPLICATE_CHECKING_IGNORES_ATTRIBUTES =
+    Set.new((%w(created_at updated_at id) + [DUPLICATE_MARK_COLUMN]).map(&:to_sym))
+
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  def duplicate?
+    duplicate_of.present?
+  end
+  alias_method :marked_as_duplicate?, :duplicate?
+  alias_method :slave?, :duplicate?
+  
+  def master?
+    !slave?
+  end
+  
+  # Return the ultimate master for a record, which may be the record itself.
+  def progenitor
+    parent = self
+    seen = Set.new
+
+    loop do
+      return parent if parent.master?
+      raise DuplicateCheckingError, "Loop detected in duplicates chain at #{parent.class}##{parent.id}" if seen.include?(parent)
+      seen << parent
+      parent = parent.duplicate_of
+    end
+  end
+  
+  # Return either an Array of exact duplicates for this record, or nil if no exact duplicates were found.
+  #
+  # Note that this method requires that all associations are set before this method is called.
+  def find_exact_duplicates
+    matchable_attributes = attributes.reject { |key, value|
+      self.class.duplicate_checking_ignores_attributes.include?(key.to_sym)
+    }
+    duplicates = self.class.where(matchable_attributes).reject{|t| t.id == id}
+    duplicates.blank? ? nil : duplicates
+  end
+
+  module ClassMethods
+    def self.extended(klass)
+      klass.instance_eval do
+        cattr_accessor(:_duplicate_checking_ignores_attributes) { Set.new }
+        cattr_accessor(:_duplicate_squashing_ignores_associations) { Set.new }
+
+        belongs_to :duplicate_of, :class_name => name, :foreign_key => DUPLICATE_MARK_COLUMN
+        has_many   :duplicates,   :class_name => name, :foreign_key => DUPLICATE_MARK_COLUMN
+
+        scope :marked_duplicates, -> { where("#{table_name}.#{DUPLICATE_MARK_COLUMN} IS NOT NULL") }
+        scope :non_duplicates, -> { where("#{table_name}.#{DUPLICATE_MARK_COLUMN} IS NULL") }
+      end
+    end
+
+    # Return set of attributes that should be ignored for duplicate checking
+    def duplicate_checking_ignores_attributes(*args)
+      _duplicate_checking_ignores_attributes.merge(args.map(&:to_sym)) unless args.empty?
+      DUPLICATE_CHECKING_IGNORES_ATTRIBUTES + _duplicate_checking_ignores_attributes
+    end
+
+    # Return set of associations that will be ignored during duplicate squashing
+    def duplicate_squashing_ignores_associations(*args)
+      _duplicate_squashing_ignores_associations.merge(args.map(&:to_sym)) unless args.empty?
+      _duplicate_squashing_ignores_associations
+    end
+
+    # Return events with duplicate values for a given set of fields.
+    #
+    # Options:
+    # * :grouped => Return Hash of events grouped by commonality, rather than returning an Array. Defaults to false.
+    # * :where => String that specifies additional arguments to add to the WHERE clause.
+    # * :select => String that specified additional arguments to add to the SELECT clause.
+    # * :from => String that specifies additional arguments to add to the FROM clause
+    # * :joins => String that specifies additional argument to add to a JOINS clause.
+    def find_duplicates_by(fields, options={})
+      DuplicateFinder.new(self, fields, options).find
+    end
+
+    # Squash duplicates. Options accept ActiveRecord instances or IDs.
+    #
+    # Options:
+    # :duplicates => ActiveRecord instance(s) to mark as duplicates
+    # :master => ActiveRecord instance to use as master
+    def squash(master, duplicates)
+      DuplicateSquasher.new(master, duplicates, name.downcase).squash
+    end
+  end
+end
+
+class DuplicateCheckingError < Exception
+end
+
+end

data/lib/calagator/duplicate_checking/controller_actions.rb

@@ -0,0 +1,38 @@
+module Calagator
+
+module DuplicateChecking
+  module ControllerActions
+    # GET /#{model_class}/duplicates
+    def duplicates
+      @type = params[:type]
+      @grouped_venues = @grouped_events = model_class.find_duplicates_by_type(@type)
+    rescue ArgumentError => e
+      @grouped_venues = @grouped_events = {}
+      flash[:failure] = e.to_s
+    end
+
+    # POST /#{model_class}/squash_multiple_duplicates
+    def squash_many_duplicates
+      master = model_class.find_by_id(params[:master_id])
+      duplicate_ids = params.keys.grep(/^duplicate_id_\d+$/){|t| params[t].to_i}
+      duplicates = model_class.where(id: duplicate_ids)
+
+      squasher = model_class.squash(master, duplicates)
+      if squasher.success
+        flash[:success] = squasher.success
+      else
+        flash[:failure] = squasher.failure
+      end
+      redirect_to action: "duplicates", type: params[:type]
+    end
+
+    private
+
+    def model_class
+      # Derive model class from controller name
+      "Calagator::#{controller_name.singularize.titleize}".constantize
+    end
+  end
+end
+
+end

data/lib/calagator/duplicate_checking/duplicate_finder.rb

@@ -0,0 +1,83 @@
+module Calagator
+
+module DuplicateChecking
+  class DuplicateFinder < Struct.new(:model, :fields, :options)
+    def find
+      scope = model.select("a.*")
+      scope = scope.from("#{model.table_name} a, #{model.table_name} b")
+      scope = scope.where(options[:where]) if options[:where]
+      scope = scope.where("a.id <> b.id")
+      scope = scope.where("a.duplicate_of_id" => nil)
+      scope = scope.where(query)
+      scope.distinct
+
+      records = scope.all
+      records = group_by_fields(records) if grouped
+      records
+    end
+
+    def fields
+      super || :all
+    end
+
+    private
+
+    def query
+      case fields
+        when :all then query_from_all
+        when :any then query_from_any
+        else query_from_fields
+      end
+    end
+
+    def grouped
+      options[:grouped] || false
+    end
+
+    def group_by_fields records
+      # Group by the field values we're matching on; skip any values for which we only have one record
+      records = records.group_by do |record|
+        fields.map do |field|
+          record.read_attribute(field)
+        end
+      end
+      records.reject { |value, group| group.size <= 1 }
+    end
+
+    def query_from_all
+      attributes.map do |attr|
+        "((a.#{attr} = b.#{attr}) OR (a.#{attr} IS NULL AND b.#{attr} IS NULL))"
+      end.join(" AND ")
+    end
+
+    def query_from_any
+      attributes.map do |attr|
+        query = "(a.#{attr} = b.#{attr} AND ("
+        column = model.columns.find {|column| column.name.to_sym == attr}
+        case column.type
+        when :integer, :decimal
+          query << "a.#{attr} != 0 AND "
+        when :string, :text
+          query << "a.#{attr} != '' AND "
+        end
+        query << "a.#{attr} IS NOT NULL))"
+      end.join(" OR ")
+    end
+
+    def query_from_fields
+      raise ArgumentError, "Unknown fields: #{fields.inspect}" if (Array(fields) - attributes).any?
+      Array(fields).map do |attr|
+        "a.#{attr} = b.#{attr}"
+      end.join(" AND ")
+    end
+
+    def attributes
+      # TODO make find_duplicates_by(:all) pay attention to ignore fields
+      model.new.attribute_names.map(&:to_sym).reject do |attr|
+        [:id, :created_at, :updated_at, :duplicate_of_id, :version].include?(attr)
+      end
+    end
+  end
+end
+
+end

data/lib/calagator/duplicate_checking/duplicate_squasher.rb

@@ -0,0 +1,74 @@
+module Calagator
+
+module DuplicateChecking
+  class DuplicateSquasher < Struct.new(:master, :duplicates, :model_name, :failure, :success)
+    def duplicates
+      Array(super)
+    end
+
+    def valid?
+      name = model_name.split("::").last
+      self.failure = "A master #{name} must be selected." if master.blank?
+      self.failure = "At least one duplicate #{name} must be selected." if duplicates.empty?
+      self.failure = "The master #{name} could not be squashed into itself." if duplicates.include?(master)
+      failure.blank?
+    end
+
+    def squash
+      if valid?
+        duplicates.each do |duplicate|
+          SingleSquasher.new(master, duplicate, model_name).squash
+        end
+        name = model_name.split("::").last
+        self.success = "Squashed duplicate #{name.pluralize} #{duplicates.map(&:title)} into master #{master.id}."
+      end
+      self
+    end
+
+    class SingleSquasher < Struct.new(:master, :duplicate, :model_name)
+      def squash
+        # Transfer any venues that use this now duplicate venue as a master
+        if duplicate.duplicates.any?
+          DuplicateSquasher.new(master, duplicate.duplicates, model_name).squash
+        end
+
+        squash_associations
+
+        duplicate.update_attribute(:duplicate_of, master)
+        duplicate
+      end
+
+      private
+
+      def squash_associations
+        # Transfer any has_many associations of this model to the master
+        master.class.reflect_on_all_associations(:has_many).each do |association|
+          next if association.name == :duplicates
+          next if master.class.duplicate_squashing_ignores_associations.include?(association.name)
+
+          # Handle tags - can't simply reassign, need to be unique, and they may have some of the same tags
+          if association.name == :tag_taggings
+            squash_tags
+          else
+            squash_association association
+          end
+        end
+      end
+
+      # custom behavior for tags, concatentate the two objects tag strings together
+      def squash_tags
+        master.tag_list = master.tag_list + duplicate.tag_list
+        master.save_tags
+      end
+
+      def squash_association(association)
+        foreign_objects = duplicate.send(association.name)
+        foreign_objects.each do |object|
+          object.update_attribute(association.foreign_key, master.id)
+        end
+      end
+    end
+  end
+end
+
+end