buoy_data 0.2.0 → 1.0.0.beta.0

data/Rakefile

@@ -12,6 +12,8 @@ begin
     gem.authors = ["Adam Weller"]
     gem.add_development_dependency 'rspec'
     gem.add_dependency 'httparty'
+    gem.add_dependency 'json'
+    gem.add_dependency 'nokogiri'
     gem.files.include 'lib/buoy_data/*.rb'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end

data/VERSION

@@ -1 +1 @@
-0.2.0
+1.0.0.beta.0

data/buoy_data.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{buoy_data}
-  s.version = "0.2.0"
+  s.version = "1.0.0.beta.0"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adam Weller"]
-  s.date = %q{2010-11-22}
+  s.date = %q{2011-03-20}
   s.description = %q{The goal of this gem is to provide marine buoy data from a variety of sources}
   s.email = %q{minch@trazzler.com}
   s.extra_rdoc_files = [
@@ -25,9 +25,15 @@ Gem::Specification.new do |s|
      "VERSION",
      "buoy_data.gemspec",
      "lib/buoy_data.rb",
-     "lib/buoy_data/buoy.rb",
-     "lib/buoy_data/noaa_buoy.rb",
-     "spec/noaa_buoy_spec.rb",
+     "lib/buoy_data/buoy_list.rb",
+     "lib/buoy_data/buoy_reading.rb",
+     "lib/buoy_data/noaa_buoy_forecast.rb",
+     "lib/buoy_data/noaa_buoy_list.rb",
+     "lib/buoy_data/noaa_buoy_observation.rb",
+     "lib/buoy_data/noaa_buoy_reading.rb",
+     "spec/noaa_buoy_forecast_spec.rb",
+     "spec/noaa_buoy_list_spec.rb",
+     "spec/noaa_buoy_observation_spec.rb",
      "spec/spec_helper.rb"
   ]
   s.homepage = %q{http://github.com/minch/buoy_data}
@@ -36,7 +42,9 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{Fetch marine buoy data from various sources}
   s.test_files = [
-    "spec/noaa_buoy_spec.rb",
+    "spec/noaa_buoy_forecast_spec.rb",
+     "spec/noaa_buoy_list_spec.rb",
+     "spec/noaa_buoy_observation_spec.rb",
      "spec/spec_helper.rb"
   ]
 
@@ -47,13 +55,19 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 0"])
       s.add_runtime_dependency(%q<httparty>, [">= 0"])
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 0"])
       s.add_dependency(%q<httparty>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 0"])
     s.add_dependency(%q<httparty>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
   end
 end
 

data/lib/buoy_data/buoy_list.rb

@@ -0,0 +1,66 @@
+module BuoyData
+  require 'open-uri'
+  require 'nokogiri'
+
+  class BuoyList
+    def get
+      stats = {
+        :stations => [],
+        :errors => [] 
+      }
+      @doc = doc
+      @station_list = stations doc
+
+      @station_list.each do |s|
+        begin
+          h = scrape_station s
+          stats[:stations].push h
+        rescue => e
+          stats[:errors].push({ :url => s, :error => e })
+        end
+      end
+
+      stats
+    end
+
+    def url
+    end
+
+    def base_url
+    end
+
+    def doc
+      doc = Nokogiri::HTML(open(url))  
+    end
+
+    private
+    def stations(doc)
+    end
+
+    def scrape_station(url)
+    end
+
+    def lat_lng_from(text)
+      text = text.sub(/ \(.*$/, '')
+      matches = /([0-9\.]+ ?[NESW]) ([0-9\.]+ ?[NESW])/.match(text)
+
+      matches && matches.size == 3 ? [ $1, $2 ] : []
+    end
+
+    def normal_lat(lat)
+      lat, dir = lat.split(/ /)
+      lat = lat.to_f
+      lat = -lat if dir == 'S'
+
+      lat
+    end
+
+    def normal_lng(lng)
+      lng, dir = lng.split(/ /)
+      lng = lng.to_f
+      lng = -lng if dir == 'W'
+
+      lng
+    end
+  end
+end

data/lib/buoy_data/{buoy.rb → buoy_reading.rb}

@@ -1,5 +1,5 @@
 module BuoyData
-  class Buoy
+  class BuoyReading
     include HTTParty
 
     GET_SUCCESS = 200

data/lib/buoy_data/noaa_buoy_forecast.rb

@@ -0,0 +1,149 @@
+module BuoyData
+  class NoaaBuoyForecast < NoaaBuoyReading
+    require 'json/add/core'
+
+    def get(raw = true)
+      response = super raw
+      return unless response
+      @response = response
+
+      @parsed_response = parse_response @response
+    end
+
+    # This pattern differs from the previous design but seems better
+    def to_json
+      response = @parsed_response
+      response = response.map{|row| bull_row_to_hash row}
+      response.to_json
+    end
+
+    private
+
+    def bull_row_to_hash(row)
+      hash = {
+        :time => { :day => row[0][0], :hour => row[0][1] },
+        :total => { :hst => row[1][0], :n => row[1][1], :x => row[1][2] }
+      }
+      5.times do |index|
+        hash.store("separate_wave_#{index}".to_sym, separate_wave_hash(row[2..row.size-1]))
+      end
+      hash
+    end
+
+    def separate_wave_hash(array)
+      h = {
+        :wind => array[0],
+        :hs => array[1],
+        :tp => array[2],
+        :dir => array[3]
+      }
+    end
+
+    def parse_response(response)
+      # Get all readings
+      response = response_to_array response
+
+      # Remove header and footer
+      response = remove_header_from response
+      response = remove_footer_from response
+
+      # Now really go through this gnarly bull format and make some sense of it
+      response = parse_bull_response response
+
+      response
+    end
+
+    def parse_bull_response(response)
+      parsed_response = []
+
+      #
+      # Split on pipes so we can get the swell breakdown
+      #
+      response = response.map{|row| row.split(/\|/)}
+
+      # The first column is always empty
+      response = response.map{|row| row[1..row.size-1]}
+      
+      #
+      # Now we have eight columns
+      #
+      # 1: day and hour
+      # 2: Total Significant wave height and exclusions (Hst, n and x)
+      # 3-7: Up to 6 swell breakdowns (* Hs, Tp, dir)
+      #
+      # Where:
+      #
+      # Hst : Total significant wave height.
+      # n   : Number of fields with Hs > 0.05 in 2-D spectrum.
+      # x   : Number of fields with Hs > 0.15 not in table.
+      # Hs  : Significant wave height of separate wave field.
+      # Tp  : Peak period of separate wave field.
+      # dir : Mean direction of separate wave field.
+      # *   : Wave generation due to local wind probable.
+      #
+
+      days_and_hours = response.map{|row| row.first.strip.split(/\s+/)}
+
+      #
+      # totals
+      #
+      totals = response.map do |row|
+        row = row[1].strip.split(/\s+/)
+        row = row.size == 3 ? row : row << ""
+      end
+
+      #
+      # up to 6 swells
+      #
+      separate_waves = response.map{|row| row[2..7].map{|col| parse_separate_wave(col)}}
+
+      # build the parsed_response
+      num = response.size-1
+      num.times do |index|
+        row = []
+
+        row.push days_and_hours[index]
+        row.push totals[index]
+        separate_waves[index].each do |sw|
+          row.push sw
+        end
+
+        parsed_response.push row
+      end
+
+      parsed_response
+    end
+
+    # Example: "* 1.3 5.3 298"
+    def parse_separate_wave(separate_wave)
+      # There's probably a slicker way but...
+      separate_wave = separate_wave.split(/\s+/)
+
+      # The first column, the wind probable '*', may or may not be present
+      # In either cases we have an empty string as the first field
+      separate_wave.size == 5 ? separate_wave.shift : separate_wave
+
+      separate_wave
+    end
+
+    def base_url(buoy_id = @buoy_id)
+      @buoy_id ||= buoy_id
+
+      # NOTE:  The bull files will probably change per ocean/geo:
+      # E.g., there are two model runs for the northern atlantic:
+      #"http://polar.ncep.noaa.gov/waves/latest_run/nah.#{buoy_id}.bull"
+
+      "http://polar.ncep.noaa.gov/waves/latest_run/wna.#{buoy_id}.bull"
+    end
+
+    # The header is the first 7 lines
+    def remove_header_from(response)
+      response[7..response.size-1]
+    end
+
+    # The footer is the last 8 lines
+    def remove_footer_from(response)
+      response[0..response.size-11]
+    end
+  end
+end

data/lib/buoy_data/noaa_buoy_list.rb

@@ -0,0 +1,49 @@
+module BuoyData
+  class NoaaBuoyList < BuoyList
+    def url
+      "#{base_url}/to_station.shtml"
+    end
+
+    def base_url
+      'http://www.ndbc.noaa.gov'
+    end
+
+    private
+    def stations(doc)
+      text = "National Data Buoy Center Stations"
+      xpath = "//h4[text()='#{text}']/following-sibling::pre/a"
+
+      doc.xpath(xpath).map{|element| element['href']}
+    end
+
+    def scrape_station(url)
+      url = [base_url, url].join('/')
+      doc = Nokogiri::HTML(open(url))  
+      h = {}
+      xpath = "//h1"
+
+      # Title, Station Id and Description
+      title = doc.xpath(xpath).text
+      h[:url] = url
+      h[:title] = title
+      station_id, description = title.split(/ - /)
+      station_id = station_id.sub(/station ?/i, '')
+      h[:station_id] = station_id
+      h[:description] = description
+
+      # Lat and Lng
+      xpath += "/following-sibling::table/tr/td/p/b"
+      elements = doc.xpath(xpath)
+      elements = elements.map(&:text)
+      element = elements.find{|e| /\d{2,3}\.\d{2,3}/.match(e)}
+      latlng = lat_lng_from element
+
+      unless latlng.empty?
+        h[:lat] = normal_lat latlng.first
+        h[:lng] = normal_lng latlng.last
+      end
+
+      h
+    end
+  end
+end

data/lib/buoy_data/{noaa_buoy.rb → noaa_buoy_observation.rb}

@@ -1,34 +1,53 @@
 module BuoyData
-  class NoaaBuoy < Buoy
-    format :plain
+  class NoaaBuoyObservation < NoaaBuoyReading
+    require 'json/add/core'
 
-    attr_accessor :buoy_id, :url, :response
+    def get_all(format = :plain)
+      response = get(true)
+      return unless response
+      response = response_to_array response
 
-    def initialize(buoy_id, unit_system = self.class.default_unit_system)
-      @url = base_url(buoy_id)
-      @unit_system = unit_system
-    end
+      case format
+      when :json
+        json_response = []
 
-    def metaclass
-      class << self
-        self
-      end
-    end
+        fields = fields_from_response response
+        # Remvoe the field rows
+        response = response[2..response.size-1]
+
+        response.each do |reading|
+          reading = reading.split
+          json = reading_to_hash(reading, fields).to_json
+          json_response.push json
+        end
 
-    def get(raw = false)
-      response = self.class.get(@url)
-      return unless response and response.code == GET_SUCCESS
-      return response if raw
+        response = json_response
+      end
 
-      @response = response
-      parse_response
+      response
     end
 
-    def get_all
-      response = get(true)
+    def parse_response(response)
+      # Get all readings
+      response = response_to_array response
+      # The first line are the fields
+      fields = fields_from_response response
+
+      # The second line are different names of the fields
+      # The third line is the most recent reading
+      response = most_recent_from_response response
       return unless response
 
-      response.parsed_response.split(/\n/)
+      # Set instance vars per field so we can use dot notation to reference each
+      hash = reading_to_hash(response, fields)
+      define_attributes(hash)
+    end
+
+    def define_attributes(hash)
+      hash.each_pair { |key, value|
+        metaclass.send :attr_accessor, key
+        send "#{key}=".to_sym, value
+      }
     end
 
     #
@@ -72,54 +91,5 @@ module BuoyData
 
       "http://www.ndbc.noaa.gov/data/realtime2/#{buoy_id}.spec"
     end
-
-    def self.source
-      :noaa
-    end
-
-    def self.google_chart_base(buoy_id)
-      url = [ "http://chart.apis.google.com/chart?cht=lc" ]
-      url << "chtt=#{buoy_id}"
-      url << "chts=666666,12"
-      url << "chco=3366FF"
-      url << "chs=250x100"
-      url << "chds=0,10"
-      url << "chl=WVHT".gsub(/ /, '%20')
-      url.join('&')
-    end
-
-
-    def self.default_unit_system
-      :metric
-    end
-
-    def parse_response
-      # Get all readings
-      response = @response.parsed_response.split(/\n/)
-
-      # The first line are the fields
-      fields = response.first.sub(/^#/, '').split
-
-      # The second line are different names of the fields
-
-      # The third line is the most recent reading
-      response = response[2].split rescue nil
-      return unless response
-
-      # Set instance vars per field so we can use dot notation to reference each
-      hash = {}
-      fields.each_with_index do |field, index|
-        hash.store(field, response[index])
-      end
-
-      define_attributes(hash)
-    end
-
-    def define_attributes(hash)
-      hash.each_pair { |key, value|
-        metaclass.send :attr_accessor, key
-        send "#{key}=".to_sym, value
-      }
-    end
   end
 end

data/lib/buoy_data/noaa_buoy_reading.rb

@@ -0,0 +1,69 @@
+module BuoyData
+  class NoaaBuoyReading < BuoyReading
+    format :plain
+
+    attr_accessor :buoy_id, :url, :response
+
+    def initialize(buoy_id, unit_system = self.class.default_unit_system)
+      @url = base_url(buoy_id)
+      @unit_system = unit_system
+    end
+
+    def metaclass
+      class << self
+        self
+      end
+    end
+
+    def get(raw = false)
+      response = self.class.get(@url)
+      return unless response and response.code == GET_SUCCESS
+      return response if raw
+
+      @response = response
+      parse_response(response)
+    end
+
+    def self.source
+      :noaa
+    end
+
+    def self.google_chart_base(buoy_id)
+      url = [ "http://chart.apis.google.com/chart?cht=lc" ]
+      url << "chtt=#{buoy_id}"
+      url << "chts=666666,12"
+      url << "chco=3366FF"
+      url << "chs=250x100"
+      url << "chds=0,10"
+      url << "chl=WVHT".gsub(/ /, '%20')
+      url.join('&')
+    end
+
+
+    def self.default_unit_system
+      :metric
+    end
+
+    private
+    def fields_from_response(response)
+      fields = response.first.sub(/^#/, '').split
+    end
+
+    def response_to_array(response)
+      response.parsed_response.split(/\n/)
+    end
+
+    def most_recent_from_response(response)
+      response[2].split rescue nil
+    end
+
+    def reading_to_hash(reading, fields)
+      hash = {}
+      fields.each_with_index do |field, index|
+        hash.store(field, reading[index])
+      end
+
+      hash
+    end
+  end
+end

data/lib/buoy_data.rb

@@ -1,3 +1,7 @@
 require 'httparty'
-require 'buoy_data/buoy'
-require 'buoy_data/noaa_buoy'
+require 'buoy_data/buoy_list'
+require 'buoy_data/buoy_reading'
+require 'buoy_data/noaa_buoy_reading'
+require 'buoy_data/noaa_buoy_list'
+require 'buoy_data/noaa_buoy_observation'
+require 'buoy_data/noaa_buoy_forecast'

data/spec/noaa_buoy_forecast_spec.rb

@@ -0,0 +1,28 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe BuoyData::NoaaBuoyForecast do
+  before(:each) do
+    #FakeWeb.allow_net_connect = false # TODO
+  end
+
+  let (:noaa_buoy_forecast) { BuoyData::NoaaBuoyForecast.new(41012) }
+
+  context 'get' do
+    before(:each) do
+      noaa_buoy_forecast.get
+    end
+
+    it 'should get buoy data' do
+      noaa_buoy_forecast.response.code.should == BuoyData::NoaaBuoyForecast::GET_SUCCESS
+    end
+
+    it 'should get buoy data as json' do
+      noaa_buoy_forecast.get
+      json = noaa_buoy_forecast.to_json
+      json.should be
+      # We really should get down and dirty here, thoroughly checking the fields
+      hash = JSON.parse(json).first
+      hash.should be_a Hash
+    end
+  end
+end

data/spec/noaa_buoy_list_spec.rb

@@ -0,0 +1,19 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe BuoyData::NoaaBuoyList do
+  let (:buoy_list) { BuoyData::NoaaBuoyList.new }
+
+  context 'get' do
+    it "should get buoy list" do
+      buoy_list.should be
+      buoy_list.stub(:stations).and_return(stubbed_station_list)
+      stations = buoy_list.get
+
+      stations.should_not be_empty
+    end
+  end
+
+  def stubbed_station_list
+    [ "station_page.php?station=41012", "station_page.php?station=sauf1" ]
+  end
+end

data/spec/noaa_buoy_observation_spec.rb

@@ -0,0 +1,57 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe BuoyData::NoaaBuoyObservation do
+  before(:each) do
+    #FakeWeb.allow_net_connect = false # TODO
+  end
+
+  let (:noaa_buoy_observation) { BuoyData::NoaaBuoyObservation.new(41012) }
+
+  context 'get' do
+    it 'should get buoy data' do
+      noaa_buoy_observation.get
+      noaa_buoy_observation.response.code.should == BuoyData::NoaaBuoyObservation::GET_SUCCESS
+    end
+
+    it 'should respond to noaa fields' do
+      noaa_buoy_observation.get
+      noaa_buoy_observation.should respond_to :WVHT
+    end
+
+    it 'should generate google chart url' do
+      noaa_buoy_observation.google_chart_url.should match(/chart.apis.google.com/)
+    end
+
+    it 'should parse date fields' do
+      noaa_buoy_observation.get
+      noaa_buoy_observation.YY.should match(/^\d{4}$/)
+      noaa_buoy_observation.DD.should match(/^\d{2}$/)
+      noaa_buoy_observation.hh.should match(/^\d{2}$/)
+      noaa_buoy_observation.mm.should match(/^\d{2}$/)
+    end
+  end
+
+  context 'get_all' do
+    it 'should get all' do
+      response = noaa_buoy_observation.get_all
+      response.should be_a Array
+      response.size.should > 1
+    end
+
+    it 'should get all as :json' do
+      json = noaa_buoy_observation.get_all(:json)
+      hash = JSON.parse(json.first)
+      hash.should be_a(Hash)
+    end 
+
+    it 'should parse date fields' do
+      response = noaa_buoy_observation.get_all(:json)
+      noaa_buoy_observation = JSON.parse(response.first)
+
+      noaa_buoy_observation[:YY.to_s].should match(/^\d{4}$/)
+      noaa_buoy_observation[:DD.to_s].should match(/^\d{2}$/)
+      noaa_buoy_observation[:hh.to_s].should match(/^\d{2}$/)
+      noaa_buoy_observation[:mm.to_s].should match(/^\d{2}$/)
+    end
+  end
+end

metadata

@@ -1,12 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: buoy_data
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  prerelease: true
   segments: 
+  - 1
+  - 0
   - 0
-  - 2
+  - beta
   - 0
-  version: 0.2.0
+  version: 1.0.0.beta.0
 platform: ruby
 authors: 
 - Adam Weller
@@ -14,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-22 00:00:00 -05:00
+date: 2011-03-20 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -43,6 +45,32 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: The goal of this gem is to provide marine buoy data from a variety of sources
 email: minch@trazzler.com
 executables: []
@@ -61,9 +89,15 @@ files:
 - VERSION
 - buoy_data.gemspec
 - lib/buoy_data.rb
-- lib/buoy_data/buoy.rb
-- lib/buoy_data/noaa_buoy.rb
-- spec/noaa_buoy_spec.rb
+- lib/buoy_data/buoy_list.rb
+- lib/buoy_data/buoy_reading.rb
+- lib/buoy_data/noaa_buoy_forecast.rb
+- lib/buoy_data/noaa_buoy_list.rb
+- lib/buoy_data/noaa_buoy_observation.rb
+- lib/buoy_data/noaa_buoy_reading.rb
+- spec/noaa_buoy_forecast_spec.rb
+- spec/noaa_buoy_list_spec.rb
+- spec/noaa_buoy_observation_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
 homepage: http://github.com/minch/buoy_data
@@ -85,11 +119,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: 
@@ -98,5 +134,7 @@ signing_key:
 specification_version: 3
 summary: Fetch marine buoy data from various sources
 test_files: 
-- spec/noaa_buoy_spec.rb
+- spec/noaa_buoy_forecast_spec.rb
+- spec/noaa_buoy_list_spec.rb
+- spec/noaa_buoy_observation_spec.rb
 - spec/spec_helper.rb

data/spec/noaa_buoy_spec.rb

@@ -1,28 +0,0 @@
-require File.join(File.dirname(__FILE__), 'spec_helper')
-
-describe BuoyData::NoaaBuoy do
-  before(:each) do
-    #FakeWeb.allow_net_connect = false # TODO
-    @noaa_buoy = BuoyData::NoaaBuoy.new(41114)
-  end
-
-  context 'get' do
-    it 'should get buoy data' do
-      @noaa_buoy.get
-      @noaa_buoy.response.code.should == BuoyData::NoaaBuoy::GET_SUCCESS
-    end
-
-    it 'should respond to noaa fields' do
-      @noaa_buoy.get
-      @noaa_buoy.should respond_to :WVHT
-    end
-
-    it 'should get all buoy data' do
-      @noaa_buoy.get_all.size.should >= 1
-    end
-
-    it 'should get all buoy data' do
-      @noaa_buoy.google_chart_url.should match(/chart.apis.google.com/)
-    end
-  end
-end