GeoScraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5caa254cac2c48a1048d3c0d9432603fa24ce81d
+  data.tar.gz: f9df9458ad9c2eba5a2d800ce7f2e5869508862c
+SHA512:
+  metadata.gz: b3f469bfe1a36ba5c6c92beb8eabf1a2d2fd6f1ea69442bcb52cbbc3266c1ae2e471774f62ae7a838894aba6f890bcec899edac67897e1a7c848c48fbc256f03
+  data.tar.gz: f6e4fe8fb6cd71eaffe37e31d967fdd81a3a0fc4b4a8a4f11a44e308e3478b9126480e5d87bc284746753840062b50a6bf89d4f0cd6463ada3f36ed116804bb6

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in GeoScraper.gemspec
+gemspec

data/GeoScraper.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'GeoScraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "GeoScraper"
+  spec.version       = GeoScraper::VERSION
+  spec.authors       = ["isamu arimoto"]
+  spec.email         = ["isamu@to-kyo.to"]
+  spec.description   = %q{GeoScraper is location data (latitude/longitude) scraper.}
+  spec.summary       = %q{GeoScraper is location data (latitude/longitude) scraper.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"  
+
+  spec.add_dependency "nokogiri"
+
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 isamu arimoto
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# GeoScraper
+
+GeoScraper is location data (latitude/longitude) scraper.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'GeoScraper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install GeoScraper
+
+## Usage
+
+    gs = GeoScraper.new(uri)
+    gs.lon # get longitude
+    gs.lat # get latitude
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+task :default => :spec

data/lib/GeoScraper.rb

@@ -0,0 +1,8 @@
+require "GeoScraper/version"
+require "GeoScraper/core"
+require "GeoScraper/foursquare"
+require "GeoScraper/yelp"
+require "GeoScraper/gnavi"
+require "GeoScraper/tabelog"
+require "GeoScraper/sanzeromin"
+

data/lib/GeoScraper/core.rb

@@ -0,0 +1,44 @@
+require "open-uri"
+require "nokogiri"
+
+class GeoScraper
+  @@pattern = []
+
+  @url = ""
+  @matches = nil
+  @result = nil
+
+  def initialize url
+    @url = url
+    @@pattern.each do |pattern|
+      if @matches = pattern[0].match(@url)
+        extend Object.const_get("GeoScraper::" + pattern[1])
+        @matches = Regexp.last_match
+        @result = scrape
+        break
+      end
+    end
+  end
+
+  def get_doc uri
+    Nokogiri::HTML.parse(open(uri, "User-Agent" => "Mozilla/5.0 Geo Scraper").read)
+  end
+
+  def has_data?
+    !@result.nil?
+  end
+
+  def lon
+    if  has_data?
+      @result["longitude"].to_f
+    end
+  end
+
+  def lat
+    if  has_data?
+      @result["latitude"].to_f
+    end
+  end
+
+end
+

data/lib/GeoScraper/foursquare.rb

@@ -0,0 +1,18 @@
+class GeoScraper
+  @@pattern << [/https?:\/\/(\w+\.)?foursquare.com\//, "FourSquare"]
+
+  module FourSquare
+    def scrape
+      doc = get_doc @url
+      meta = doc.css('meta').inject({}) {|s, v|
+        s[v.attr('property')] ||= v.attr('content')
+        s[v.attr('name')] ||= v.attr('content')
+        s
+      }
+      {
+        "longitude" => meta["playfoursquare:location:longitude"],
+        "latitude" => meta["playfoursquare:location:latitude"]
+      }
+    end
+  end
+end

data/lib/GeoScraper/gnavi.rb

@@ -0,0 +1,24 @@
+class GeoScraper
+  @@pattern << [/http:\/\/r\.gnavi\.co\.jp\/(\w+)\//, "Gnavi"]
+
+  module Gnavi
+    def scrape
+      uri = "http://r.gnavi.co.jp/#{@matches[1]}/map/"
+      
+      doc = get_doc(uri)
+      if figureCanvas = doc.xpath("//div[@id='figureCanvas']")
+        value = figureCanvas[0].attributes["class"].value
+        geo = value.split(" ").inject({}){|s, v|
+          values = v.split(":")
+          s[values[0]] = values[1]
+          s
+        }
+      end
+      { 
+        "longitude" => geo["lng"],
+        "latitude" => geo["lat"]
+      }
+    end
+    
+  end
+end

data/lib/GeoScraper/sanzeromin.rb

@@ -0,0 +1,25 @@
+class GeoScraper
+  @@pattern << [/^http:\/\/30min.jp\/place/, "SanzeroMin"]
+
+  module SanzeroMin
+    def scrape
+      doc = get_doc @url
+
+      lon = nil
+      lat = nil
+      doc.css("script").select{|script|
+        script.inner_text.match(/SZAreaMatch/)
+      }[0].inner_text.split("\n").each {|line| 
+        if line =~ /SZAreaMatch\.longitude/ 
+          lon = line.scan(/^\-|\d[\d\-\.]+/)[0]
+        elsif line =~ /SZAreaMatch\.latitude/
+          lat = line.scan(/^\-|\d[\d\-\.]+/)[0]
+        end
+      }
+      {
+        "longitude" => lon,
+        "latitude" => lat
+      }
+    end
+  end
+end

data/lib/GeoScraper/tabelog.rb

@@ -0,0 +1,27 @@
+class GeoScraper
+  @@pattern << [/http:\/\/tabelog\.com(\/[A-Za-z\d]+){4}/, "Tabelog"]
+  
+  module Tabelog
+    def scrape
+      url = @matches[0] + "/dtlmap/"
+      doc = get_doc url
+
+      lon = nil
+      lat = nil
+      doc.css("script").select{|script|
+        script.inner_text.match(/LatLng/)
+      }[0].inner_text.split("\n").each {|line| 
+        if line =~ /var\ lng/ 
+          lon = line.scan(/^\-|\d[\d\-\.]+/)[0]
+        elsif line =~ /var\ lat/
+          lat = line.scan(/^\-|\d[\d\-\.]+/)[0]
+        end
+      }
+      {
+        "longitude" => lon,
+        "latitude" => lat
+      }
+
+    end
+  end
+end

data/lib/GeoScraper/version.rb

@@ -0,0 +1,3 @@
+class GeoScraper
+  VERSION = "0.0.1"
+end

data/lib/GeoScraper/yelp.rb

@@ -0,0 +1,19 @@
+class GeoScraper
+  @@pattern << [/http:\/\/www\.yelp\.com\//, "Yelp"]
+  
+  module Yelp
+    def scrape
+      doc = get_doc @url
+
+      meta = doc.css('meta').inject({}) {|s, v|
+        s[v.attr('property')] ||= v.attr('content')
+        s[v.attr('name')] ||= v.attr('content')
+        s
+      }
+      {
+        "longitude" => meta['place:location:longitude'],
+        "latitude" => meta['place:location:latitude']
+      }
+    end
+  end
+end

data/spec/GeoScraper_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+describe GeoScraper do
+  before do
+  end
+
+  it 'should get geo data from foursquare' do
+    uri = "https://ja.foursquare.com/v/%E6%9D%B1%E4%BA%AC%E3%83%89%E3%83%BC%E3%83%A0-tokyo-dome/4b7544c3f964a520a8032ee3"
+    gs = GeoScraper.new(uri)
+    
+    gs.lon.should == 139.7519087791443
+    gs.lat.should == 35.705584611645335
+  end
+  
+  it 'should get geo data from yelp' do
+    uri = "http://www.yelp.com/biz/greenhearts-family-farm-csa-san-francisco"
+    gs = GeoScraper.new(uri)
+    
+    gs.lon.should == -122.4355626
+    gs.lat.should == 37.7733623
+  end
+
+  it 'should get geo data from 30min' do
+    gs = GeoScraper.new "http://30min.jp/place/5561"
+    
+    gs.lon.should == 139.78297
+    gs.lat.should == 35.684971
+  end
+
+  it 'should get geo data from gnavi' do
+    gs = GeoScraper.new "http://r.gnavi.co.jp/g398542/map/"
+    
+    gs.lon.should == 139.77357
+    gs.lat.should == 35.627817
+  end  
+
+  it 'should get geo data from tabelog' do
+    uri = "http://tabelog.com/tokyo/A1311/A131102/13003661/dtlmap/"
+    gs = GeoScraper.new(uri)
+    
+    gs.lon.should == 139.795076693823
+    gs.lat.should == 35.7104300850057
+  end
+
+  it 'should get geo data from tabelog no map url' do
+    uri = "http://tabelog.com/tokyo/A1311/A131102/13003661/"
+    gs = GeoScraper.new(uri)
+    
+    gs.lon.should == 139.795076693823
+    gs.lat.should == 35.7104300850057
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'GeoScraper'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: GeoScraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- isamu arimoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: GeoScraper is location data (latitude/longitude) scraper.
+email:
+- isamu@to-kyo.to
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- GeoScraper.gemspec
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/GeoScraper.rb
+- lib/GeoScraper/core.rb
+- lib/GeoScraper/foursquare.rb
+- lib/GeoScraper/gnavi.rb
+- lib/GeoScraper/sanzeromin.rb
+- lib/GeoScraper/tabelog.rb
+- lib/GeoScraper/version.rb
+- lib/GeoScraper/yelp.rb
+- spec/GeoScraper_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: GeoScraper is location data (latitude/longitude) scraper.
+test_files:
+- spec/GeoScraper_spec.rb
+- spec/spec_helper.rb