buoy_data 1.0.0.beta.1 → 1.0.0.beta.2

data/VERSION

@@ -1 +1 @@
-1.0.0.beta.1
+1.0.0.beta.2

data/buoy_data.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{buoy_data}
-  s.version = "1.0.0.beta.1"
+  s.version = "1.0.0.beta.2"
 
   s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adam Weller"]
-  s.date = %q{2011-04-17}
+  s.date = %q{2011-05-06}
   s.description = %q{The goal of this gem is to provide marine buoy data from a variety of sources}
   s.email = %q{minch@trazzler.com}
   s.extra_rdoc_files = [
@@ -27,14 +27,18 @@ Gem::Specification.new do |s|
      "lib/buoy_data.rb",
      "lib/buoy_data/buoy_list.rb",
      "lib/buoy_data/buoy_reading.rb",
+     "lib/buoy_data/field_map.rb",
+     "lib/buoy_data/noaa.rb",
      "lib/buoy_data/noaa_buoy_forecast.rb",
      "lib/buoy_data/noaa_buoy_list.rb",
      "lib/buoy_data/noaa_buoy_observation.rb",
      "lib/buoy_data/noaa_buoy_reading.rb",
+     "lib/buoy_data/noaa_field_map.rb",
      "lib/buoy_data/noaa_station.rb",
      "spec/noaa_buoy_forecast_spec.rb",
      "spec/noaa_buoy_list_spec.rb",
      "spec/noaa_buoy_observation_spec.rb",
+     "spec/noaa_station_spec.rb",
      "spec/spec_helper.rb"
   ]
   s.homepage = %q{http://github.com/minch/buoy_data}
@@ -46,6 +50,7 @@ Gem::Specification.new do |s|
     "spec/noaa_buoy_forecast_spec.rb",
      "spec/noaa_buoy_list_spec.rb",
      "spec/noaa_buoy_observation_spec.rb",
+     "spec/noaa_station_spec.rb",
      "spec/spec_helper.rb"
   ]
 

data/lib/buoy_data/buoy_list.rb

@@ -3,7 +3,7 @@ module BuoyData
   require 'nokogiri'
 
   class BuoyList
-    def get
+    def get(limit = 0)
       stats = {
         :stations => [],
         :station_count => 0,
@@ -13,15 +13,20 @@ module BuoyData
       @doc = doc
       @station_list = stations doc
 
+      # Only relevant for testing
+      @station_list = @station_list[0..limit] if limit > 0
+
       @station_list.each do |station_url|
-        begin
+        #p station_url
+        
+        #begin
           h = scrape_station(station_url)
           stats[:stations].push h
           stats[:station_count] += 1
-        rescue => e
-          stats[:error_count] += 1
-          stats[:errors].push({ :url => s, :error => e.backtrace })
-        end
+        #rescue => e
+          #stats[:error_count] += 1
+          #stats[:errors].push({ :url => station_url, :error => e.backtrace })
+        #end
       end
 
       stats

data/lib/buoy_data/field_map.rb

@@ -0,0 +1,3 @@
+class FieldMap
+  # TODO:  these classes should be singletons
+end

data/lib/buoy_data/noaa.rb

@@ -0,0 +1,5 @@
+module BuoyData
+  module Noaa
+    BASE_URL = 'http://www.ndbc.noaa.gov'
+  end
+end

data/lib/buoy_data/noaa_buoy_list.rb

@@ -5,7 +5,11 @@ module BuoyData
     end
 
     def base_url
-      'http://www.ndbc.noaa.gov'
+      Noaa::BASE_URL
+    end
+
+    def scrape_station(url)
+      NoaaStation.new.scrape(url)
     end
 
     private
@@ -15,10 +19,5 @@ module BuoyData
 
       doc.xpath(xpath).map{|element| element['href']}
     end
-
-    def scrape_station(url)
-      url = [ base_url, url ].join('/')
-      NoaaStation.new.scrape(url)
-    end
   end
 end

data/lib/buoy_data/noaa_field_map.rb

@@ -0,0 +1,15 @@
+class NoaaFieldMap < FieldMap
+  def self.field_map
+    f = {
+      :wvht => ['Wave Height', 'ft'],
+      :dpd => ['Dominant Wave Period', 's'],
+      :apd => ['Average Period', 's'],
+      :gst => ['Wind Gust', 'kts'],
+      :wspd => ['Wind Speed', 'kts'],
+      :wdir => ['Wind Direction', 'deg'],
+      #:wtmp => ['Water Temperature',"#{String.new('U+00B0')} F"],
+      :wtmp => ['Water Temperature',"F"],
+      :atmp => ['Air Temperature', "F"]
+    } 
+  end
+end

data/lib/buoy_data/noaa_station.rb

@@ -2,8 +2,23 @@ module BuoyData
   require 'open-uri'
   require 'nokogiri'
 
+  # This class scrapes readings from noaa stations.
+  # A station in this parlance is an noaa resource that does
+  # not have wave data (in other words is not necessarily
+  # a floating buoy).
+  #
+  # There is no super class for this model as we don't yet
+  # know if this set up will apply for other data sources.
+  #
+  # NOTE: This class can currently be used to scrape the latest
+  # readings for buoys as well.  The reason it exists however
+  # is that we can't fetch data for the stations from the .bull
+  # files (they don't exists for stations only buoys).
+
   class NoaaStation
     def scrape(url)
+      url = normalize_url url
+
       h = {}
       xpath = "//h1"
 
@@ -27,29 +42,83 @@ module BuoyData
       element = elements.find{|e| /\d{2,3}\.\d{2,3}/.match(e)}
       latlng = lat_lng_from element
 
-      unless latlng.empty?
+      unless latlng.blank?
         h[:lat] = normal_lat latlng.first
         h[:lng] = normal_lng latlng.last
       end
 
       # Latest reading
-      xpath = "//table/caption[@class='titleDataHeader'][text()[contains(.,'Conditions')]]"
+      current_reading = {}
+
+      current_reading = latest_reading doc
+
+      # If we couldn't get a water temp from this station then it's not of use
+      return {} unless valid_reading? current_reading
+      h.merge! current_reading
+
+      h
+    end
+
+    # Do the best we can but get something
+    def latest_reading(doc)
+      # Check the current reading
+      reading = current_reading doc
+
+      # If we don't have a wave height (:wvht) then let's check the previous reading
+      unless reading[:wvht]
+        p_reading = previous_reading doc
+        reading = p_reading if p_reading[:wvht]
+      end
+
+      reading
+    end
+
+    # Reding from the 'Conditions at..as of..' table
+    def current_reading(doc)
+      reading = {}
+
+      xpath = "//table/caption[@class='titleDataHeader']["
+      xpath += "contains(text(),'Conditions')"
+      xpath += " and "
+      xpath += "not(contains(text(),'Solar Radiation'))"
+      xpath += "]"
+
+      # Get the reading timestamp
+      source_updated_at = reading_timestamp(doc, xpath)
+      reading[:source_updated_at] = source_updated_at
+
+      # Get the reading data
       xpath += "/../tr"
       elements = doc.xpath xpath
 
-      current_reading = {}
       unless elements.empty?
         elements.each do |element|
-          reading = scrape_condition_from_element(element)  
-          current_reading.merge! reading unless reading.blank?
+          r = scrape_condition_from_element(element)
+          reading.merge! r unless r.blank?
         end
       end
 
-      # If we couldn't get a water temp from this station then it's not of use
-      return {} unless valid_reading? current_reading
-      h.merge! current_reading
+      reading
+    end
 
-      h
+    # Most recent observation from the 'Previous observations' table
+    # This is unfinished, will need to capture the markup when the
+    # current reading is not avaliable and stub it w/fakeweb...
+    def previous_reading(doc)
+      reading = {}
+      text = 'Previous observations'
+      xpath = "//table/caption[@class='dataHeader'][text()[contains(.,'#{text}')]]"
+
+      xpath += "/../tr/td"
+      elements = doc.xpath xpath
+
+      #unless elements.empty?
+        #elements.each do |element|
+          #p element.text
+        #end
+      #end
+
+      reading
     end
 
     private
@@ -60,10 +129,7 @@ module BuoyData
 
     def scrape_condition_from_element(element)
       reading = {}
-      # (include units)
-      #regexp = /\(([A-Z]+)\):\n\t ?(.*)\n$/
-      # (no units)
-      regexp = /\(([A-Z]+)\):\n\t ?([0-9.-]+).*\n$/
+      regexp = /\(([A-Z]+)\):\n\n?\t? ?(.*)\n$/m
       text = element.text
 
       if regexp.match(text)
@@ -72,6 +138,7 @@ module BuoyData
         key = key.downcase.to_sym
         value = $2
         return reading unless value
+        value = $1 if value =~ /([0-9.]+)/
         value = value.lstrip.rstrip.to_f
         reading = { key => value }
       end
@@ -80,7 +147,8 @@ module BuoyData
     end
 
     def lat_lng_from(text)
-      text = text.sub(/ \(.*$/, '')
+      text = text.sub(/ \(.*$/, '') rescue nil
+      return unless text
       matches = /([0-9\.]+ ?[NESW]) ([0-9\.]+ ?[NESW])/.match(text)
 
       matches && matches.size == 3 ? [ $1, $2 ] : []
@@ -101,5 +169,34 @@ module BuoyData
 
       lng
     end
+
+    def normalize_url(url)
+      base_url = Noaa::BASE_URL
+      /^#{base_url}/.match(url) ? url : [Noaa::BASE_URL, url].join('/')
+    end
+
+    def logger
+      RAILS_DEFAULT_LOGGER if defined? RAILS_DEFAULT_LOGGER
+    end
+
+    def reading_timestamp(doc, xpath)
+      elements = doc.xpath(xpath).children rescue []
+      unless elements.empty?
+        begin
+          elements = elements.to_a
+          elements.reject!{|e|e.name == 'br'}
+          s = elements.last.text.sub(/ ?:$/, '')
+          time, date = s.split(/ on /)
+          month, day, year = date.split(/\//)
+          date = [ day, month, year ].join('-')
+          hour = time[0..1]
+          minutes = time[2..3]
+          time = [ hour, minutes ].join(':')
+          s = [ date, time ].join('T')
+          t = Time.parse s if s
+        rescue
+        end
+      end
+    end
   end
 end

data/lib/buoy_data.rb

@@ -1,8 +1,11 @@
 require 'httparty'
 require 'buoy_data/buoy_list'
 require 'buoy_data/buoy_reading'
+require 'buoy_data/noaa'
 require 'buoy_data/noaa_buoy_reading'
 require 'buoy_data/noaa_station'
 require 'buoy_data/noaa_buoy_list'
 require 'buoy_data/noaa_buoy_observation'
 require 'buoy_data/noaa_buoy_forecast'
+require 'buoy_data/field_map'
+require 'buoy_data/noaa_field_map'

data/spec/noaa_buoy_list_spec.rb

@@ -13,11 +13,6 @@ describe BuoyData::NoaaBuoyList do
       results[:errors].should be_empty
       stations = results[:stations]
       stations.compact.should_not be_empty
-      station = stations.first
-
-      # Verify we have water temp value
-      wtmp = station[:wtmp]
-      wtmp.should be_kind_of Float
     end
   end
 

data/spec/noaa_station_spec.rb

@@ -0,0 +1,22 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe BuoyData::NoaaStation do
+  let (:station) { BuoyData::NoaaStation.new }
+
+  it "should scrape station" do
+    data = station.scrape station_url
+    data.should_not be_empty
+    data.should be_a(Hash)
+
+    data[:wtmp].should be_a(Float)
+    data[:source_updated_at].should be_a(Time)
+  end
+
+  def station_url
+    "station_page.php?station=sauf1"
+  end
+
+  def buoy_url
+    "station_page.php?station=41012"
+  end
+end

metadata

@@ -7,8 +7,8 @@ version: !ruby/object:Gem::Version
   - 0
   - 0
   - beta
-  - 1
-  version: 1.0.0.beta.1
+  - 2
+  version: 1.0.0.beta.2
 platform: ruby
 authors: 
 - Adam Weller
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-17 00:00:00 -04:00
+date: 2011-05-06 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -91,14 +91,18 @@ files:
 - lib/buoy_data.rb
 - lib/buoy_data/buoy_list.rb
 - lib/buoy_data/buoy_reading.rb
+- lib/buoy_data/field_map.rb
+- lib/buoy_data/noaa.rb
 - lib/buoy_data/noaa_buoy_forecast.rb
 - lib/buoy_data/noaa_buoy_list.rb
 - lib/buoy_data/noaa_buoy_observation.rb
 - lib/buoy_data/noaa_buoy_reading.rb
+- lib/buoy_data/noaa_field_map.rb
 - lib/buoy_data/noaa_station.rb
 - spec/noaa_buoy_forecast_spec.rb
 - spec/noaa_buoy_list_spec.rb
 - spec/noaa_buoy_observation_spec.rb
+- spec/noaa_station_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
 homepage: http://github.com/minch/buoy_data
@@ -138,4 +142,5 @@ test_files:
 - spec/noaa_buoy_forecast_spec.rb
 - spec/noaa_buoy_list_spec.rb
 - spec/noaa_buoy_observation_spec.rb
+- spec/noaa_station_spec.rb
 - spec/spec_helper.rb