calagator 0.0.1.pre1

data/app/models/calagator/source/parser/hcal.rb

@@ -0,0 +1,108 @@
+# == Source::Parser::Hcal
+#
+# Reads hCalendar events.
+
+vendored_mofo_dir = File.expand_path("../../../../../vendor/gems/mofo-0.2.8/lib", File.dirname(__FILE__))
+$: << vendored_mofo_dir
+require "mofo"
+
+module Calagator
+
+class Source::Parser::Hcal < Source::Parser
+  self.label = :hCalendar
+
+  EVENT_TO_HCALENDAR_FIELD_MAP = {
+    :title => :summary,
+    :description => :description,
+    :start_time => :dtstart,
+    :end_time => :dtend,
+    :url => :url,
+    :venue => :location,
+  }
+
+  def to_events
+    hcals.map do |hcal|
+      event = Event.new
+      event.source = opts[:source]
+      EVENT_TO_HCALENDAR_FIELD_MAP.each do |field, mofo_field|
+        next unless hcal.respond_to?(mofo_field)
+        next unless value = decoded_field(hcal, mofo_field)
+        event.send "#{field}=", value
+      end
+      event_or_duplicate(event)
+    end.uniq do |event|
+      [event.attributes, event.venue.try(:attributes)]
+    end
+  end
+
+  private
+
+  def decoded_field(hcal, mofo_field)
+    return unless raw_field = hcal.send(mofo_field)
+    decoded_field = case mofo_field
+    when :dtstart
+      HTMLEntities.new.decode(raw_field)
+    when :location
+      to_venue(opts.merge(:value => raw_field))
+    else
+      raw_field
+    end
+  end
+
+  VENUE_TO_HCARD_FIELD_MAP = {
+    :title => :fn,
+    :telephone => :tel,
+    :email => :email,
+    :description => :note,
+  }
+
+  # Return a Venue.
+  #
+  # Options:
+  # * :value -- hCard or string location
+  def to_venue(opts)
+    venue = Venue.new
+    venue.source = opts[:source]
+    case raw = opts[:value]
+    when String
+      venue.title = raw
+    when HCard
+      assign_fields(venue, raw)
+      assign_geo(venue, raw) if raw.respond_to?(:geo)
+      assign_address(venue, raw) if raw.respond_to?(:adr)
+    end
+    venue.geocode!
+    venue_or_duplicate(venue)
+  end
+
+  def assign_fields(venue, raw)
+    VENUE_TO_HCARD_FIELD_MAP.each do |field, mofo_field|
+      venue[field] = raw.send(mofo_field).try(:strip_html) if raw.respond_to?(mofo_field)
+    end
+  end
+
+  def assign_geo(venue, raw)
+    %w(latitude longitude).each do |field|
+      venue[field] = raw.geo.send(field) if raw.geo.respond_to?(field)
+    end
+  end
+
+  def assign_address(venue, raw)
+    attributes = %w(street_address locality region country_name postal_code).reduce({}) do |attributes, field|
+      attributes[field] = raw.adr.send(field) if raw.adr.respond_to?(field)
+      attributes
+    end
+
+    attributes["country"] = attributes.delete("country_name")
+    attributes["postal_code"] = attributes["postal_code"].to_s if attributes["postal_code"]
+    venue.attributes = attributes
+  end
+
+  def hcals
+    content = self.class.read_url(opts[:url])
+    something = hCalendar.find(:text => content)
+    something.is_a?(hCalendar) ? [something] : something
+  end
+end
+
+end

data/app/models/calagator/source/parser/http_authentication_required_error.rb

@@ -0,0 +1,6 @@
+# Exception raised if user requests parsing of a URL that requires
+# authentication but none was provided.
+class Calagator::Source::Parser
+  class HttpAuthenticationRequiredError < Exception
+  end
+end

data/app/models/calagator/source/parser/ical.rb

@@ -0,0 +1,186 @@
+# == Source::Parser::Ical
+#
+# Reads iCalendar events.
+#
+# Example:
+#   events = Source::Parser::Ical.to_events('http://appendix.23ae.com/calendars/AlternateHolidays.ics')
+#
+# Sample sources:
+#   webcal://appendix.23ae.com/calendars/AlternateHolidays.ics
+#   http://appendix.23ae.com/calendars/AlternateHolidays.ics
+module Calagator
+
+class Source::Parser::Ical < Source::Parser
+  self.label = :iCalendar
+
+  VENUE_CONTENT_RE = /^BEGIN:VVENUE$.*?^END:VVENUE$/m
+
+  # Override Base::read_url to handle "webcal" scheme addresses.
+  def self.read_url(url)
+    super(url.gsub(/^webcal:/, 'http:'))
+  end
+
+  def to_events
+    return false unless calendars = content_calendars
+
+    events = calendars.flat_map do |calendar|
+      calendar.events.map do |component|
+        next if skip_old? and old?(component)
+        component_to_event(component, calendar)
+      end
+    end
+
+    events.compact.uniq do |event|
+      [event.attributes, event.venue.try(:attributes)]
+    end
+  end
+
+  private
+
+  def skip_old?
+    # Skip old events by default
+    true unless opts[:skip_old] == false
+  end
+
+  def old?(component)
+    cutoff = Time.now.yesterday
+    (component.dtend || component.dtstart).to_time < cutoff
+  end
+
+  def content_calendars
+    content = self.class.read_url(opts[:url]).gsub(/\r\n/, "\n")
+    content = munge_gmt_dates(content)
+    RiCal.parse_string(content)
+  rescue Exception => e
+    if e.message =~ /Invalid icalendar file/
+      false # Invalid data, give up.
+    else
+      raise e # Unknown error, we should care.
+    end
+  end
+
+  def component_to_event(component, calendar)
+    event = Event.new({
+      source:      opts[:source],
+      title:       component.summary,
+      description: component.description,
+      url:         component.url,
+    })
+
+    dates_for_tz(component, event)
+
+    event.venue = to_venue(content_venue(component, calendar), opts.merge(fallback: component.location))
+    event_or_duplicate(event)
+  end
+
+  def content_venue(component, calendar)
+    content_venues = calendar.to_s.scan(VENUE_CONTENT_RE)
+
+    # finding the event venue id - VVENUE=V0-001-001423875-1@eventful.com
+    venue_uid = component.location_property.params["VVENUE"]
+    # finding in the content_venues array an item matching the uid
+    venue_uid ? content_venues.find{|content_venue| content_venue.match(/^UID:#{venue_uid}$/m)} : nil
+  rescue Exception => e
+    # Ignore
+    Rails.logger.info("Source::Parser::Ical.to_events : Failed to parse content_venue for event -- #{e}")
+    nil
+  end
+
+  # Helper to set the start and end dates correctly depending on whether it's a floating or fixed timezone
+  def dates_for_tz(component, event)
+    if component.dtstart_property.tzid
+      event.start_time  = component.dtstart
+      event.end_time    = component.dtend
+    else
+      event.start_time  = Time.zone.parse(component.dtstart_property.value)
+      if component.dtend_property
+        event.end_time = Time.zone.parse(component.dtend_property.value)
+      else
+        if component.duration
+          event.end_time = component.duration_property.add_to_date_time_value(event.start_time)
+        else
+          event.end_time = event.start_time
+        end
+      end
+    end
+  end
+
+  def munge_gmt_dates(content)
+    content.gsub(/;TZID=GMT:(.*)/, ':\1Z')
+  end
+
+  # Return an Venue extracted from an iCalendar input.
+  #
+  # Arguments:
+  # * value - String with iCalendar data to parse which contains a VVENUE item.
+  #
+  # Options:
+  # * :fallback - String to use as the title for the location if the +value+ doesn't contain a VVENUE.
+  def to_venue(value, opts={})
+    venue = Venue.new
+
+    # VVENUE entries are considered just Vcards,
+    # treating them as such.
+    if vcard_hash = vcard_hash_from_value(value)
+      venue.attributes = {
+        title:          vcard_hash['NAME'],
+        street_address: vcard_hash['ADDRESS'],
+        locality:       vcard_hash['CITY'],
+        region:         vcard_hash['REGION'],
+        postal_code:    vcard_hash['POSTALCODE'],
+        country:        vcard_hash['COUNTRY'],
+      }
+      venue.latitude, venue.longitude = vcard_hash['GEO'].split(/;/).map(&:to_f)
+
+    elsif opts[:fallback].present?
+      venue.title = opts[:fallback]
+    else
+      return nil
+    end
+
+    venue.geocode!
+    venue_or_duplicate(venue)
+  end
+
+  def vcard_hash_from_value(value)
+    value ||= ""
+    return unless data = value.scan(VENUE_CONTENT_RE).first
+
+    # Only use first vcard of a VVENUE
+    vcard = RiCal.parse_string(data).first
+
+    # Extract all properties, including non-standard ones, into an array of "KEY;meta-qualifier:value" strings
+    vcard_lines = vcard.export_properties_to(StringIO.new(''))
+
+    # Turn a String-like object into an Enumerable.
+    vcard_lines = vcard_lines.respond_to?(:lines) ? vcard_lines.lines : vcard_lines
+
+    hash_from_vcard_lines(vcard_lines)
+  end
+
+  # Return hash parsed from VCARD lines.
+  #
+  # Arguments:
+  # * vcard_lines - Array of "KEY;meta-qualifier:value" strings.
+  def hash_from_vcard_lines(vcard_lines)
+    vcard_lines.reduce({}) do |vcard_hash, vcard_line|
+      if matcher = vcard_line.match(/^([^;]+?)(;[^:]*?)?:(.*)$/)
+        _, key, qualifier, value = *matcher
+
+        if qualifier
+          # Add entry for a key and its meta-qualifier
+          vcard_hash["#{key}#{qualifier}"] = value
+
+          # Add fallback entry for a key from the matching meta-qualifier, e.g. create key "foo" from contents of key with meta-qualifier "foo;bar".
+          vcard_hash[key] ||= value
+        else
+          # Add entry for a key without a meta-qualifier.
+          vcard_hash[key] = value
+        end
+      end
+      vcard_hash
+    end
+  end
+end
+
+end

data/app/models/calagator/source/parser/meetup.rb

@@ -0,0 +1,72 @@
+module Calagator
+
+class Source::Parser::Meetup < Source::Parser
+  self.label = :Meetup
+  self.url_pattern = %r{^http://(?:www\.)?meetup\.com/[^/]+/events/([^/]+)/?}
+
+  def to_events
+    return fallback unless SECRETS.meetup_api_key.present?
+    return unless data = get_data
+    event = Event.new({
+      source:      opts[:source],
+      title:       data['name'],
+      description: data['description'],
+      url:         data['event_url'],
+      venue:       to_venue(data['venue']),
+      tag_list:    "meetup:event=#{data['event_id']}, meetup:group=#{data['group']['urlname']}",
+      # Meetup sends us milliseconds since the epoch in UTC
+      start_time:  Time.at(data['time']/1000).utc,
+    })
+
+    [event_or_duplicate(event)]
+  end
+
+  private
+
+  def fallback
+    to_events_wrapper(
+      Source::Parser::Ical,
+      %r{^http://(?:www\.)?meetup\.com/([^/]+)/events/([^/]+)/?},
+      lambda { |matcher| "http://www.meetup.com/#{matcher[1]}/events/#{matcher[2]}/ical" }
+    )
+  end
+
+  def get_data
+    to_events_api_helper(opts[:url], "problem") do |event_id|
+      [
+        "https://api.meetup.com/2/event/#{event_id}",
+        {
+          key: SECRETS.meetup_api_key,
+          sign: 'true'
+        }
+      ]
+    end
+  end
+
+  def to_venue(value)
+    return if value.blank?
+    venue = Venue.new({
+      source: opts[:source],
+      title: value['name'],
+      street_address: [value['address_1'], value['address_2'], value['address_3']].compact.join(", "),
+      locality: value['city'],
+      region: value['state'],
+      postal_code: value['zip'],
+      country: value['country'],
+      telephone: value['phone'],
+      tag_list: "meetup:venue=#{value['id']}",
+    })
+    venue.geocode!
+    venue_or_duplicate(venue)
+  end
+
+  def to_events_wrapper(driver, source, target)
+    if matcher = opts[:url].try(:match, source)
+      url = target.call(matcher)
+      opts[:content] = self.class.read_url(url)
+      driver.new(opts).to_events
+    end
+  end
+end
+
+end

data/app/models/calagator/source/parser/not_found.rb

@@ -0,0 +1,9 @@
+class Calagator::Source::Parser
+  # == Source::Parser::NotFound
+  #
+  # Exception thrown to indicate that the source isn't found and no other parsers should be tried.
+  #
+  # This exception should only be thrown by the canonical handler that can definitively tell if the source doesn't exist. For example, the Source::Parser::Facebook parser can throw this exception when given a http://facebook.com/ URLs because it can be sure whether these exist.
+  class NotFound < StandardError
+  end
+end

data/app/models/calagator/source/parser/plancast.rb

@@ -0,0 +1,59 @@
+module Calagator
+
+class Source::Parser::Plancast < Source::Parser
+  self.label = :Plancast
+  self.url_pattern = %r{^http://(?:www\.)?plancast\.com/p/([^/]+)/?}
+
+  def to_events
+    return unless data = get_data
+    event = Event.new({
+      source:      opts[:source],
+      title:       data['what'],
+      description: data['description'],
+
+      # Plancast is sending floating times as Unix timestamps, which is hard to parse
+      start_time:  ActiveSupport::TimeWithZone.new(nil, Time.zone, Time.at(data['start'].to_i).utc),
+      end_time:    ActiveSupport::TimeWithZone.new(nil, Time.zone, Time.at(data['stop'].to_i).utc),
+
+      url:         (data['external_url'] || data['plan_url']),
+      tag_list:    "plancast:plan=#{data['event_id']}",
+
+      venue:       to_venue(data['place'], data['where']),
+    })
+
+    [event_or_duplicate(event)]
+  end
+
+  private
+
+  def get_data
+    to_events_api_helper(opts[:url]) do |event_id|
+      [
+        'http://api.plancast.com/02/plans/show.json',
+        {
+          plan_id: event_id,
+          extensions: 'place'
+        }
+      ]
+    end
+  end
+
+  def to_venue(value, fallback=nil)
+    value = "" if value.nil?
+    if value.present?
+      venue = Venue.new({
+        source: opts[:source],
+        title: value['name'],
+        address: value['address'],
+        tag_list: "plancast:place=#{value['id']}",
+      })
+      venue.geocode!
+      venue_or_duplicate(venue)
+    elsif fallback.present?
+      venue = Venue.new(title: fallback)
+      venue_or_duplicate(venue)
+    end
+  end
+end
+
+end

data/app/models/calagator/venue.rb

@@ -0,0 +1,161 @@
+# == Schema Information
+# Schema version: 20110604174521
+#
+# Table name: venues
+#
+#  id              :integer         not null, primary key
+#  title           :string(255)
+#  description     :text
+#  address         :string(255)
+#  url             :string(255)
+#  created_at      :datetime
+#  updated_at      :datetime
+#  street_address  :string(255)
+#  locality        :string(255)
+#  region          :string(255)
+#  postal_code     :string(255)
+#  country         :string(255)
+#  latitude        :decimal(, )
+#  longitude       :decimal(, )
+#  email           :string(255)
+#  telephone       :string(255)
+#  source_id       :integer
+#  duplicate_of_id :integer
+#  version         :integer
+#  closed          :boolean
+#  wifi            :boolean
+#  access_notes    :text
+#  events_count    :integer
+#
+require "calagator/decode_html_entities_hack"
+require "calagator/strip_whitespace"
+require "calagator/url_prefixer"
+require "paper_trail"
+require "loofah-activerecord"
+require "loofah/activerecord/xss_foliate"
+
+module Calagator
+
+class Venue < ActiveRecord::Base
+  self.table_name = "venues"
+
+  attr_protected
+
+  include StripWhitespace
+
+  has_paper_trail
+  acts_as_taggable
+
+  xss_foliate :sanitize => [:description, :access_notes]
+  include DecodeHtmlEntitiesHack
+
+  # Associations
+  has_many :events, dependent: :nullify
+  def future_events; events.future_with_venue; end
+  def past_events; events.past_with_venue; end
+  belongs_to :source
+
+  # Triggers
+  strip_whitespace! :title, :description, :address, :url, :street_address, :locality, :region, :postal_code, :country, :email, :telephone
+  before_save :geocode!
+
+  # Validations
+  validates_presence_of :title
+  validates_format_of :url,
+    :with => /(http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/,
+    :allow_blank => true,
+    :allow_nil => true
+  validates_inclusion_of :latitude,
+    :in => -90..90,
+    :allow_nil => true,
+    :message => "must be between -90 and 90"
+  validates_inclusion_of :longitude,
+    :in => -180..180,
+    :allow_nil => true,
+    :message => "must be between -180 and 180"
+
+  validates :title, :description, :address, :url, :street_address, :locality, :region, :postal_code, :country, :email, :telephone, blacklist: true
+
+  # Duplicates
+  include DuplicateChecking
+  duplicate_checking_ignores_attributes    :source_id, :version, :closed, :wifi, :access_notes
+  duplicate_squashing_ignores_associations :tags, :base_tags, :taggings
+
+  # Named scopes
+  scope :masters,          -> { where(duplicate_of_id: nil).includes(:source, :events, :tags, :taggings) }
+  scope :with_public_wifi, -> { where(wifi: true) }
+  scope :in_business,      -> { where(closed: false) }
+  scope :out_of_business,  -> { where(closed: true) }
+
+  #===[ Finders ]=========================================================
+
+  # Return Hash of Venues grouped by the +type+, e.g., a 'title'. Each Venue
+  # record will include an <tt>events_count</tt> field containing the number of
+  # events at the venue, which improves performance for displaying these.
+  def self.find_duplicates_by_type(type='na')
+    case type
+    when 'na', nil, ''
+      # The LEFT OUTER JOIN makes sure that venues without any events are also returned.
+      return { [] => \
+        self.where('venues.duplicate_of_id IS NULL').order('LOWER(venues.title)')
+      }
+    else
+      kind = %w[all any].include?(type) ? type.to_sym : type.split(',').map(&:to_sym)
+
+      return self.find_duplicates_by(kind, 
+        :grouped  => true, 
+        :where    => 'a.duplicate_of_id IS NULL AND b.duplicate_of_id IS NULL'
+      )
+    end
+  end
+
+  #===[ Search ]==========================================================
+
+  def self.search(query, opts={})
+    SearchEngine.search(query, opts)
+  end
+
+  #===[ Overrides ]=======================================================
+
+  def url=(value)
+    super UrlPrefixer.prefix(value)
+  end
+
+  #===[ Address helpers ]=================================================
+
+  # Does this venue have any address information?
+  def has_full_address?
+    [street_address, locality, region, postal_code, country].any?(&:present?)
+  end
+
+  # Display a single line address.
+  def full_address
+    if has_full_address?
+      "#{street_address}, #{locality} #{region} #{postal_code} #{country}"
+    end
+  end
+
+  #===[ Geocoding helpers ]===============================================
+
+  # Get an address we can use for geocoding
+  def geocode_address
+    full_address or address
+  end
+
+  # Return this venue's latitude/longitude location,
+  # or nil if it doesn't have one.
+  def location
+    if [latitude, longitude].all?(&:present?)
+      [latitude, longitude]
+    end
+  end
+
+  attr_accessor :force_geocoding
+
+  def geocode!
+    Geocoder.geocode(self)
+    true # Try to geocode, but don't complain if we can't.
+  end
+end
+
+end