wigle_api 1.0.0

data/.rvmrc

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p125@wigle-api"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.11.3" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    coderay (1.0.7)
+    diff-lcs (1.1.3)
+    method_source (0.8)
+    nokogiri (1.5.5)
+    pry (0.9.10)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.3.1)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.2)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.2)
+    slop (3.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  nokogiri
+  pry
+  rspec

data/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2012 Sam Stelfox
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,48 @@
+# WiGLE API
+This gem was written to make use of WiGLE.net's wireless access point data. Currently WiGLE doen't provide and official API so this gem may break if they change their interface. This was written to make use of the data for a single small scale project for a public display and is no longer actively used and I have no intention of maintaining it (though it's fairly simple).
+
+## Usage
+To make use of this you'll need to create an account on wigle.net (<http://wigle.net/gps/gps/main/register>). The following code example shows how to authenticate against the API and perform a simple search.
+
+   require "wigle\_api"
+
+   WigleApi.login("your-username", "your-password")
+   WigleApi.where(ssid: "linksys").each do |ap|
+     puts ap[:ssid] + ap[:statecode]
+   end
+
+   WigleApi.where(ssid: "linksys").offset(200).results
+
+Results are returned as an array of hashes with symbols for keys. Each hash has all of the available fields that WiGLE returns in their search view. Only 100 results will be returned at a time though you can use the "pagestart" search value or the .offset(num) chained methods to iterate through more results. There unfortunately isn't a way to count the total number of results from a query at this time.
+
+## Available Search Parameters
+
+* addresscode - Street address, max length 30
+* statecode - Two character state code
+* zipcode - Five character integer zip code, maxlength 5
+* variance - The latitude/longitude variance in degrees, values available in the form: 0.001, 0.002, 0.005, 0.010 (default), 0.020, 0.050, 0.100, 0.200
+* latrange1 - Begin latitude range, max length 14
+* latrange2 - End latitude range, max length 14
+* longrange1 - Begin longitude range, max length 14
+* longrange2 - End longitude range, max length 14
+* lastupdt - A timestamp used to filter based on the last time the access point database was updated in the format of 20010925174546
+* netid - BSSID / AP Max Address, expects either just the colon delimited vendor code or a full MAC such as either 0A:2C:EF or 0A:2C:EF:3D:25:1B, max length 14
+* ssid - SSID / Network Name, max length 50
+* freenet - Must be a freenet (Not sure what that means), set the value to on to filter with this, should be excluded completely otherwise
+* paynet - Must be a Commercial Pay Net (Not sure what that means either...), same deal with on to filter with this
+* dhcp - Must have DHCP enabled, on again to filter with this...
+* onlymine - Only search on the networks that the authenticated user has found
+* pagestart - The search result offset
+
+## Personal Notes on WiGLE
+While writing this Gem I got the distinct feeling that the operators of WiGLE.net aren't particularily friendly people. How they've managed to convince so many people to provide them with all this data is beyond me as they don't seem willing to return the favor of actually making the data available to their users.
+
+From the many forum posts requesting an API it seem's like the owners of WiGLE don't want an official way to access the data in their database for one of the reasons below:
+
+* They want to keep the data for themselves (possibly for commercial reasons)
+* They don't want to spend the time/energy coding an actual API
+* They are worried about bandwidth (and are not open to the idea of mirrors)
+* They are simply (insert insult here)
+
+Usually requests for making use of the data on the site are turned down rudely, and usually with various references to cats.
+

data/lib/wigle_api.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+# Gem requirements
+require "net/https"
+require "nokogiri"
+require "rubygems"
+require "uri"
+
+# Misc pieces of this gem
+require "wigle_api/connection"
+require "wigle_api/exceptions"
+require "wigle_api/version"
+
+# Web scraper portions
+require "wigle_api/scraper/parser"
+require "wigle_api/scraper/search"
+
+# Web API portions
+
+module WigleApi
+  WIGLE_URI       = URI.parse("https://wigle.net/")
+  WIGLE_LOGIN_URL = "/gps/gps/main/login"
+  WIGLE_QUERY_URL = "/gps/gps/main/confirmquery/"
+
+  def self.login(username, password)
+    @connection ||= WigleApi::Connection.new
+    @connection.login(username, password)
+  end
+
+  def self.where(args)
+    WigleApi::Scraper::Search.new(args)
+  end
+end
+
+# How to use the web scraper (will only return 100 results):
+# WigleApi.login("myusername", "mypassword")
+# @results = WigleApi.where(ssid: "linksys").results
+

data/lib/wigle_api/connection.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+
+module WigleApi
+  class Connection
+    def self.new(*args)
+      @@instance ||= super
+    end
+
+    def initialize
+      @connection = Net::HTTP.new(WIGLE_URI.host, WIGLE_URI.port)
+      @connection.ca_file = "/etc/pki/tls/certs/ca-bundle.crt"
+      @connection.use_ssl = true
+      @connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      
+      @logged_in = false
+    end
+
+    def login(username, password)
+      return true if @logged_in
+
+      login_form_data = {
+        "credential_0" => username,
+        "credential_1" => password,
+        "destination" => "/gps/gps/main",
+        "noexpire" => "on",
+      }
+
+      response = post(WIGLE_LOGIN_URL, login_form_data)
+
+      if response["Set-Cookie"].nil?
+        return false
+      end
+      
+      @auth_cookie = response["Set-Cookie"]
+      @logged_in = true
+    end
+
+    def search(data)
+      raise NotLoggedIn unless @logged_in
+
+      response = post(WIGLE_QUERY_URL, data)
+    end
+
+    private
+
+    def post(path, data)
+      request = Net::HTTP::Post.new(path)
+      request.initialize_http_header({
+        "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.57 Safari/537.1"
+      })
+      request.set_form_data(data)
+      request.add_field("Cookie", @auth_cookie) if @auth_cookie
+
+      @connection.start do |http|
+        http.request(request)
+      end
+    end
+  end
+end
+

data/lib/wigle_api/exceptions.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+module WigleApi
+  class NotLoggedIn < Exception
+  end
+end
+

data/lib/wigle_api/scraper/parser.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+module WigleApi
+  module Scraper
+    class Parser
+      def self.new(*args)
+        p = super
+        p.output
+      end
+
+      def initialize(html_body)
+        @parser = Nokogiri::HTML(html_body)
+
+        headers = parse_headers
+        results = parse_results
+
+        @output = results.map! { |r| Hash[headers.zip(r)] }
+      end
+
+      def output
+        @output
+      end
+
+      private
+
+      def parse_headers
+        @parser.css("th[@class='searchhead']").map do |node|
+          node.text.strip
+        end
+      end
+
+      def parse_results
+        @parser.css("tr[@class='search']").map do |sr|
+          sr.css("td").map do |v|
+            v.text.strip
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/wigle_api/scraper/search.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+
+=begin
+  Descriptions of all the valid search options and information about them:
+
+  addresscode - Street address, max length 30
+  statecode   - Two character state code
+  zipcode     - Five character integer zip code, maxlength 5
+  variance    - The latitude/longitude variance in degrees, values available in the form: 0.001, 0.002, 0.005, 0.010 (default), 0.020, 0.050, 0.100, 0.200
+  latrange1   - Begin latitude range, max length 14
+  latrange2   - End latitude range, max length 14
+  longrange1  - Begin longitude range, max length 14
+  longrange2  - End longitude range, max length 14
+  lastupdt    - A timestamp used to filter based on the last time the access point database was updated in the format of 20010925174546
+  netid       - BSSID / AP Max Address, expects either just the colon delimited vendor code or a full MAC such as either 0A:2C:EF or 0A:2C:EF:3D:25:1B, max length 14
+  ssid        - SSID / Network Name, max length 50
+  freenet     - Must be a freenet (Not sure what that means), set the value to on to filter with this, should be excluded completely otherwise
+  paynet      - Must be a Commercial Pay Net (Not sure what that means either...), same deal with on to filter with this
+  dhcp        - Must have DHCP enable, on again to filter with this...
+  onlymine    - Only search on the networks that the authenticated user has found
+  pagestart   - The search result offset
+  Query       - This is the submit button...
+=end
+
+module WigleApi
+  module Scraper
+    class Search
+      def initialize(args)
+        @search_arguments = {
+          "variance"    => "0.010",
+          "Query"       => "Query"
+        }
+
+        @search_arguments.merge!(clean(args))
+      end
+
+      def offset(num)
+        @search_arguments.merge!(clean({pagestart: num}))
+        self
+      end
+
+      def where(args)
+        @search_arguments.merge!(clean(args))
+        self
+      end
+
+      def results
+        conn = WigleApi::Connection.new
+        response = conn.search(@search_arguments)
+
+        unless [200, 301, 302, 303].include?(response.code.to_i)
+          return nil
+        end
+
+        # Parse the response, then turn the keys into symbols
+        WigleApi::Scraper::Parser.new(response.body).map do |row|
+          row.inject({}) do |results, (key, value)|
+            results[key.to_sym] = value.to_s
+            results
+          end
+        end
+      end
+
+      def each
+        results.each { |r| yield(r) }
+      end
+
+      private
+
+      def clean(options)
+        options = options.inject({}) do |opt, (key, value)|
+          opt[key.to_s] = value.to_s
+          opt
+        end
+
+        options.delete_if { |k,v| !valid_keys.include?(k) }
+      end
+
+      def valid_keys
+        [
+          "addresscode",
+          "statecode",
+          "zipcode",
+          "variance",
+          "latrange1",
+          "latrange2",
+          "longrange1",
+          "longrange2",
+          "lastupdt",
+          "netid",
+          "ssid",
+          "freenet",
+          "paynet",
+          "dhcp",
+          "onlymine",
+          "pagestart",
+        ]
+      end
+    end
+  end
+end
+

data/lib/wigle_api/version.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+module WigleApi
+  VERSION = "1.0.0"
+end
+

data/ruby-console

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Run the console with all the code loaded
+pry -I "./lib" -r "./lib/wigle_api"
+

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+require "rubygems"
+require "../lib/wigle_api"
+
+RSpec.configure do |config|
+end
+

data/wigle_api.gemspec

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+$:.push(File.expand_path(File.dirname(__FILE__) + '/lib'))
+require "wigle_api/version"
+
+Gem::Specification.new do |gem|
+  gem.name              = "wigle_api"
+  gem.version           = WigleApi::VERSION
+  gem.authors           = ["Sam Stelfox"]
+  gem.email             = ["sstelfox+rubygems@bedroomprogrammers.net"]
+  gem.homepage          = "https://github.com/sstelfox/wigle_api"
+  gem.license           = "MIT"
+
+  gem.summary           = "An 'API' for WiGLE.net"
+  gem.description       = "Provides an easy way to scrape WiGLE.net for small amounts of data."
+
+  gem.rubyforge_project = "wigle_api"
+
+  gem.files             = `git ls-files`.split($\)
+  gem.test_files        = gem.files.grep(%r{^spec/})
+  gem.executables       = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.require_paths     = ["lib"]
+
+  gem.add_runtime_dependency "nokogiri"
+  gem.add_development_dependency "rspec"
+
+  gem.required_ruby_version = '>= 1.9.2'
+end
+

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: wigle_api
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Sam Stelfox
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides an easy way to scrape WiGLE.net for small amounts of data.
+email:
+- sstelfox+rubygems@bedroomprogrammers.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- lib/wigle_api.rb
+- lib/wigle_api/connection.rb
+- lib/wigle_api/exceptions.rb
+- lib/wigle_api/scraper/parser.rb
+- lib/wigle_api/scraper/search.rb
+- lib/wigle_api/version.rb
+- ruby-console
+- spec/spec_helper.rb
+- wigle_api.gemspec
+homepage: https://github.com/sstelfox/wigle_api
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: wigle_api
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: An 'API' for WiGLE.net
+test_files:
+- spec/spec_helper.rb