meetup-scraper 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a37c4116353ce45a88a7eea716edea6945ae71a6
+  data.tar.gz: 191296acf767851f93d8653d3d6c5f7817bb2204
+SHA512:
+  metadata.gz: 8da3ae74ffd9ca26efb24a5a8fdb886e90777c81f0231a63bd6510c99a29a00261290ac0eecb72e3837e42524aba6653039833fd4975083efe9dfb67e3dc5cd1
+  data.tar.gz: ddae8658a9ba283e8fffebfd61f71b30b196737749cf04dab3d822c1694d811f006954d81e147f460344ad40ec9315f417b726efe09f2482932578c7d4233886

data/bin/meetup-scraper

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+# require_relative '../config/environment'
+
+require 'bundler/setup'
+require 'meetup_scraper'
+
+# launch cli app
+MeetupScraper::CommandLineInterface.new.run

data/lib/meetup_scraper.rb

@@ -0,0 +1,10 @@
+#require_relative './../config/environment'
+
+require 'nokogiri'
+require 'open-uri'
+
+require_relative './meetup_scraper/version'
+require_relative './meetup_scraper/cli_methods'
+require_relative './meetup_scraper/command_line_interface'
+require_relative './meetup_scraper/event'
+require_relative './meetup_scraper/scraper'

data/lib/meetup_scraper/cli_methods.rb

@@ -0,0 +1,67 @@
+class MeetupScraper::CliMethods
+
+  # capture user input returning search url
+  def get_user_input
+    puts 'Search Meetup.com for events in your local area'
+    puts 'Enter the meetup subject'
+    subject = gets.chomp
+    puts 'Enter your town'
+    town = gets.chomp
+    puts 'How many miles from your town are you willing to travel'
+    miles = gets.chomp.to_i
+    miles = 1 if miles == 0
+
+    base_url = 'https://www.meetup.com/find/events/?allMeetups=false&keywords='
+    "#{base_url}#{subject}&radius=#{miles}&userFreeForm=#{town}"
+  end
+
+  # fetch the meetup events
+  def search_meetup(url)
+    MeetupScraper::Scraper.fetch_meetup_list(url)
+  end
+
+  # create event instances
+  def create_events_from_hashes(event_hashes)
+    MeetupScraper::Event.create_from_collection(event_hashes)
+  end
+
+  # download the event's details
+  def fetch_event_details(url)
+    MeetupScraper::Scraper.fetch_event_details(url)
+  end
+
+  # print event detatils
+  def print_event(event)
+    puts '------------------------------'
+    puts "Title: #{event.title}"
+    puts "Organiser: #{event.organiser}"
+    puts "Date: #{event.date}"
+    puts "Time: #{event.time}"
+    puts "Number attending: #{event.num_attending}"
+    puts "Address: #{event.address}"
+    puts "Description: #{event.description}"
+    puts '------------------------------'
+  end
+
+  def pick_meetup_event
+      puts 'Enter the number of the event to view more details'
+      puts "Enter '0' to search again"
+      puts "To quit, enter 'exit'"
+      puts 'What would you like to do?'
+      gets.chomp
+  end
+
+  def print_events
+    MeetupScraper::Event.all.each_with_index do |event, i|
+      puts "------------------------------"
+      puts "#{i + 1}. #{event.title}"
+      puts "Organiser: #{event.organiser}"
+      puts "Date: #{event.date.slice(0, 10)}"
+      puts "Time: #{event.time}"
+      puts "Number attending: #{event.num_attending}"
+      puts "Url: #{event.url}"
+      puts "------------------------------"
+    end
+  end
+
+end

data/lib/meetup_scraper/command_line_interface.rb

@@ -0,0 +1,60 @@
+class MeetupScraper::CommandLineInterface
+  attr_reader :climethods
+
+  def initialize
+    @climethods = MeetupScraper::CliMethods.new
+  end
+
+  def run
+    url = self.climethods.get_user_input
+    event_hashes = climethods.search_meetup(url)
+    climethods.create_events_from_hashes(event_hashes)
+    self.print_meetup_events
+  end
+
+  def print_meetup_events
+    events = MeetupScraper::Event.all
+    while events.size < 1
+      puts "Sorry, no matches found, try again."
+      puts '------------------------------------'
+      self.run
+    end
+    self.climethods.print_events
+    self.print_meetup_event
+    #self.run
+  end
+
+  def print_meetup_event
+    input = self.climethods.pick_meetup_event
+    if (input.to_i >= 1 && input.to_i <= MeetupScraper::Event.all.size)
+      event = MeetupScraper::Event.all[input.to_i - 1]
+      url = event.url
+      # down load event's details, update event attributes & print event
+      updated_details = self.climethods.fetch_event_details(url)
+      updated_event = event.update_event_attributes(updated_details)
+      self.climethods.print_event(updated_event)
+      self.run_again
+    elsif input == '0'
+      self.run
+    elsif input == 'exit'
+      puts 'Good bye!'
+    else
+      puts 'Selection not recognised, try again'
+      print_meetup_event
+    end
+  end
+
+  def run_again
+    puts "Enter '1' to view listing again or '0' to search again"
+    input = gets.chomp
+    if input.to_i == 1
+      self.print_meetup_events
+    elsif input == '0'
+      self.run
+    else
+      puts 'Selection not recognised, try again' 
+      run_again
+    end
+  end
+
+end

data/lib/meetup_scraper/event.rb

@@ -0,0 +1,33 @@
+class MeetupScraper::Event
+  attr_accessor :title, :date, :time, :url, :num_attending, :organiser, :address, :description
+  @@all = []
+
+  def create_from_hash(event_hash)
+    event_hash.each do |k, v|
+      self.send("#{k}=", v)
+    end
+    save
+    self
+  end
+
+  def self.create_from_collection(event_hashes)
+    self.all.clear
+    event_hashes.collect {|event_hash| self.new.create_from_hash(event_hash)}
+  end
+
+  def self.all
+    @@all
+  end
+
+  def save
+    self.class.all << self
+  end
+
+  def update_event_attributes(event_hash)
+    event_hash.each do |k, v|
+      self.send("#{k}=", v)
+    end
+    self
+  end
+
+end

data/lib/meetup_scraper/scraper.rb

@@ -0,0 +1,56 @@
+=begin
+  url: 'https://www.meetup.com/find/events/?allMeetups=false&keywords=javascript&radius=2&userFreeform=London%2C+United+Kingdom'
+=end
+
+class MeetupScraper::Scraper
+
+  def self.fetch_meetup_list(url)
+    doc = Nokogiri::HTML(open(url))
+    items = doc.css('.searchResults .event-listing-container .event-listing')
+
+    items.collect do |item|
+      event = {}
+      title = item.css('.chunk a.event span').inner_text
+      if title != ''
+        event[:title] = title
+        event[:organiser] = item.css('.chunk .text--labelSecondary a span').inner_text
+        event[:url] = item.css('a').first.attr('href')
+        num_attending = item.css('.text--secondary .attendee-count').inner_text
+        event[:num_attending] = num_attending.strip.slice(0, 4).gsub(/[^\d]/, '')
+      end
+      time = item.css('.text--secondary time').text
+      if time != ''
+        event[:time] = time
+        event[:date] = item.css('time').attr('datetime').value
+      end
+      # event[:location] = item.css('.row-item')[1].css('.text--secondary a').text.strip
+      event
+    end
+  end
+
+  def self.fetch_event_details(url)
+    doc = Nokogiri::HTML(open(url))
+    items = doc.css('#event-content')
+    event = {}
+    items.each do |item|
+      event[:date] = item.css('#event-when-display h3').text
+      event_start_time = item.css('#event-start-time span').text
+      event_end_time = item.css('#event-end-time span').text
+      if (event_end_time != nil && event_end_time != '') && (event_start_time != nil && event_start_time != '')
+        event[:time] = "#{event_start_time} to #{event_end_time}"
+      else
+        time = item.css('#event-when-display p').text
+        event[:time] = time if time != '' || time != nil
+      end
+      address = item.css('#event-where-display .event-where-address').text.gsub(/\(map\)/, '').strip
+      address = 'Signup to view address' if address == nil || address == ''
+      event[:address] = address
+      description_text = ''
+      paragraphs = item.css('#event-description-wrap p')
+      paragraphs.each {|p| description_text += p.text.gsub(/â\u0080¢/, '') + "\n"}
+      event[:description] = description_text.strip
+    end
+    event
+  end
+
+end

data/lib/meetup_scraper/version.rb

@@ -0,0 +1,3 @@
+module MeetupScraper
+  VERSION = '0.3.0'
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: meetup-scraper
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Bill Fero
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+description: Search Meetup.com and scape the details of the latest meetups in your
+  area.
+email:
+- billfero@gmail.com
+executables:
+- meetup-scraper
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/meetup-scraper
+- lib/meetup_scraper.rb
+- lib/meetup_scraper/cli_methods.rb
+- lib/meetup_scraper/command_line_interface.rb
+- lib/meetup_scraper/event.rb
+- lib/meetup_scraper/scraper.rb
+- lib/meetup_scraper/version.rb
+homepage: https://github.com/theBoyMo/ruby-web-scraping-cli-app
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Scrape Meetup.com site.
+test_files: []