lakes 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fd4734a4458713c26f87380650f9f997b69ee47
-  data.tar.gz: 98432482a89cac25e45637a1acd4a50d4d3fd5e3
+  metadata.gz: d4e064b1da9183c688a5e5c72310404c1632ddbe
+  data.tar.gz: a45be37ae9ae49c384676ffa687541cc6d1edab9
 SHA512:
-  metadata.gz: 1e95cffa65e7e8778f7ae086dfc651d4b1f1dcd0e602b57fdad0a942745e0be6eff3e81a660020951184317811af5527d88ab477c1355df22f40c660576dadb2
-  data.tar.gz: ed6fb9047856f64ea9ab52fbda4ea31e6895132998f2e06ce4877d649136968fd2d5ac0e7431e00afd94dd206f909481d228dfc44127ebcff243ee171ecca2d6
+  metadata.gz: 74d1344a011c7d724a2f0b7f1acc074acc4328446694d52df375e35496005270ff9d44e1beefe69f88da865aa273c040f7f66c9b5b16fb77d906c1646060c8e0
+  data.tar.gz: 1dbf6042cecaa440ec9379fdf5da56dfcdc9929ee0312613946ccc6c928f2c53cce3d56e509f0e37675d9baf15fe0b60250ca22a8c0b2c837a0a10076cf70f32

data/lakes.gemspec

@@ -10,8 +10,8 @@ Gem::Specification.new do |spec|
 
   spec.summary       = 'This gem parses lake details from various government websites'
   spec.description   = 'I wrote this gem to originally parse texas lake data'
-  spec.homepage      = nil
-  spec.license       = 'mit'
+  spec.homepage      = 'https://github.com/ssherman/lakes'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.bindir        = "exe"
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency 'nokogiri', "~> 1.7"
 end

data/lib/lakes/helper.rb

@@ -0,0 +1,35 @@
+module Lakes
+  module Helper
+
+    # texas lake pages are encoded in Windows-1252 :(
+    def http_get(url)
+      uri = URI(url)
+      Net::HTTP.get(uri).encode('UTF-8', 'Windows-1252')
+    end
+
+    # texas lake websites use lots of non breaking spaces
+    def cleanup_data(value)
+      nbsp = 160.chr('UTF-8')
+      value = value.strip.gsub(nbsp, '')
+      value.empty? ? nil : value
+    end
+
+    def cleanup_raw_text(raw_text)
+      raw_text.try(:gsub, /\s+/, ' ').try(:strip)
+    end
+
+    # converts this:
+    # ../../../action/waterecords.php?WB_code=0001
+    # into this:
+    # http://tpwd.texas.gov/fishboat/fish/action/waterecords.php?WB_code=0001
+    # based on this:
+    # http://tpwd.texas.gov/fishboat/fish/recreational/lakes/abilene
+    def convert_relative_href(href, current_url)
+      relative_depth = href.split('..').count - 1
+      url_parts = current_url.split('/')
+      url_parts.slice!(-relative_depth, relative_depth)
+      fixed_href = href.gsub('../', '')
+      url_parts.join('/') + '/' + fixed_href
+    end
+  end
+end

data/lib/lakes/texas/lake_characteristics_parser.rb

@@ -1,57 +1,61 @@
-class LakeCharacteristicsParser
-  attr_reader :raw_text, :location_desc
-  attr_reader :surface_area_raw_text, :surface_area_in_acres
-  attr_reader :max_depth_raw_text, :max_depth_in_feet
-  attr_reader :year_impounded_raw_text, :year_impounded
-
-  def initialize(text)
-    @raw_text = text
-    parse
-  end
-
-  def parse
-    @location_desc = @raw_text.match(/^location:(.*)(surface area)|(surface acres)|(maximum depth|impounded):/im).captures.first
-    @surface_area_raw_text = @raw_text.match(/surface (area|acres):(.*)/i).try(:captures).try(:[], 1)
-    @max_depth_raw_text = @raw_text.match(/maximum depth:(.*)/i).try(:captures).try(:first)
-    @year_impounded_raw_text = @raw_text.match(/impounded:(.*)/im).try(:captures).try(:first)
-
-    @location_desc = cleanup_raw_text(@location_desc)
-
-    @surface_area_in_acres = cleanup_raw_text(@surface_area_raw_text)
-      .try(:match, /^([0-9,]+)/)
-      .try(:captures)
-      .try(:first)
-      .try(:delete, ',')
-      .try(:to_i)
-
-    @max_depth_in_feet = cleanup_raw_text(@max_depth_raw_text)
-      .try(:match, /^([0-9,]+)/)
-      .try(:captures)
-      .try(:first)
-      .try(:delete, ',')
-      .try(:to_i)
-
-    # need to handle bad data like Lake Fryer which is:
-    # Maximum depth: Average 13 feet, maximum 25 feet
-    if @max_depth_in_feet.nil?
-      @max_depth_in_feet = cleanup_raw_text(@max_depth_raw_text)
-        .try(:match, /maximum ([0-9,]+) feet/i)
-        .try(:captures)
-        .try(:first)
-        .try(:delete, ',')
-        .try(:to_i)
+module Lakes
+  class Texas
+    class LakeCharacteristicsParser
+      attr_reader :raw_text, :location_desc
+      attr_reader :surface_area_raw_text, :surface_area_in_acres
+      attr_reader :max_depth_raw_text, :max_depth_in_feet
+      attr_reader :year_impounded_raw_text, :year_impounded
+
+      def initialize(text)
+        @raw_text = text
+        parse
+      end
+
+      def parse
+        @location_desc = @raw_text.match(/^location:(.*)(surface area)|(surface acres)|(maximum depth|impounded):/im).captures.first
+        @surface_area_raw_text = @raw_text.match(/surface (area|acres):(.*)/i).try(:captures).try(:[], 1)
+        @max_depth_raw_text = @raw_text.match(/maximum depth:(.*)/i).try(:captures).try(:first)
+        @year_impounded_raw_text = @raw_text.match(/impounded:(.*)/im).try(:captures).try(:first)
+
+        @location_desc = cleanup_raw_text(@location_desc)
+
+        @surface_area_in_acres = cleanup_raw_text(@surface_area_raw_text)
+          .try(:match, /^([0-9,]+)/)
+          .try(:captures)
+          .try(:first)
+          .try(:delete, ',')
+          .try(:to_i)
+
+        @max_depth_in_feet = cleanup_raw_text(@max_depth_raw_text)
+          .try(:match, /^([0-9,]+)/)
+          .try(:captures)
+          .try(:first)
+          .try(:delete, ',')
+          .try(:to_i)
+
+        # need to handle bad data like Lake Fryer which is:
+        # Maximum depth: Average 13 feet, maximum 25 feet
+        if @max_depth_in_feet.nil?
+          @max_depth_in_feet = cleanup_raw_text(@max_depth_raw_text)
+            .try(:match, /maximum ([0-9,]+) feet/i)
+            .try(:captures)
+            .try(:first)
+            .try(:delete, ',')
+            .try(:to_i)
+        end
+
+        @year_impounded = cleanup_raw_text(@year_impounded_raw_text)
+          .try(:match, /([0-9,]+)/)
+          .try(:captures)
+          .try(:first)
+          .try(:delete, ',')
+          .try(:to_i)
+
+      end
+
+      def cleanup_raw_text(raw_text)
+        raw_text.try(:gsub, /\s+/, ' ').try(:strip)
+      end
     end
-
-    @year_impounded = cleanup_raw_text(@year_impounded_raw_text)
-      .try(:match, /([0-9,]+)/)
-      .try(:captures)
-      .try(:first)
-      .try(:delete, ',')
-      .try(:to_i)
-
-  end
-
-  def cleanup_raw_text(raw_text)
-    raw_text.try(:gsub, /\s+/, ' ').try(:strip)
   end
 end

data/lib/lakes/texas/water_conditions_parser.rb

@@ -0,0 +1,57 @@
+require 'nokogiri'
+module Lakes
+  class Texas
+    class WaterConditionsParser
+      include Lakes::Helper
+
+      attr_reader :raw_text, :raw_text_without_whitespace
+      attr_reader :water_data_uri
+      attr_reader :conservation_pool_elevation_raw_text, :conservation_pool_elevation
+      attr_reader :conservation_pool_elevation_in_ft_msl
+      attr_reader :fluctuation_raw_text, :fluctuation
+      attr_reader :normal_clarity_raw_text, :normal_clarity
+
+      def initialize(text)
+        return if text.nil?
+        @raw_text = text
+        @raw_text_without_whitespace = text.gsub(/[\t\r\n\f]+/, '').gsub(/\s\s/, ' ')
+        parse
+      end
+
+      # <a href="http://waterdatafortexas.org/reservoirs/individual/belton">Current Lake Level</a>
+      # Conservation Pool Elevation: 594 ft. msl
+      # Fluctuation: 3-5 feet
+      # Normal Clarity: Moderate
+      def parse
+        html_doc = Nokogiri::HTML.fragment(@raw_text)
+
+        html_doc_without_whitespace_chars = Nokogiri::HTML.fragment(raw_text_without_whitespace)
+        water_data_link = html_doc_without_whitespace_chars.xpath('p/a[contains(text(), "Current Lake Level")]').first
+        @water_data_uri = water_data_link.try(:[], 'href')
+        if @water_data_uri && @water_data_uri.start_with?('http://')
+          @water_data_uri.gsub!('http://', 'https://')
+        end
+
+        text_doc = html_doc.text
+        text_doc_without_whitespace = html_doc_without_whitespace_chars.text
+
+        # so many inconsistencies in the data
+        @conservation_pool_elevation_raw_text = text_doc
+          .match(/(Conservation Pool Elevation:(.*))|(Normal water level:(.*))/i)
+          .try(:captures)
+          .try(:compact)
+          .try(:[], 1)
+
+        @conservation_pool_elevation = cleanup_raw_text(
+          @conservation_pool_elevation_raw_text
+        )
+
+        @fluctuation_raw_text = text_doc.match(/Fluctuation: (.*)Normal Clarity:/im).try(:captures).try(:first)
+        @fluctuation = cleanup_raw_text(@fluctuation_raw_text)
+
+        @normal_clarity_raw_text = text_doc_without_whitespace.match(/Normal Clarity: (.*)/i).try(:captures).try(:first)
+        @normal_clarity = cleanup_raw_text(@normal_clarity_raw_text)
+      end
+    end
+  end
+end

data/lib/lakes/texas/water_data_parser.rb

@@ -0,0 +1,37 @@
+require 'nokogiri'
+module Lakes
+  class Texas
+    class WaterDataParser
+      include Lakes::Helper
+
+      attr_reader :raw_text
+      attr_reader :conservation_pool_elevation_in_ft_msl
+      attr_reader :percentage_full
+
+      def initialize(text)
+        @raw_text = text
+        #File.write("test/data/water_data/Texoma.txt", @raw_text)
+        #puts "WaterDataParser: raw_text: #{@raw_text}"
+        parse
+      end
+
+      def parse
+        html_doc = Nokogiri::HTML(@raw_text)
+        cons_pool_elevation_header_element = html_doc.xpath('//td[contains(text(), "Conservation pool elevation")]').first
+        cons_pool_elevation_root = cons_pool_elevation_header_element.try(:next_element)
+        @conservation_pool_elevation_in_ft_msl = cleanup_raw_text(cons_pool_elevation_root.try(:text))
+          .try(:match, /([0-9\.]+)/)
+          .try(:captures)
+          .try(:first)
+          .try(:to_f)
+
+        percentage_full_element = cleanup_raw_text(html_doc.css('div.page-title h2 small').try(:text))
+        @percentage_full = percentage_full_element
+          .try(:match, /^([0-9]+\.?[0-9]+)/)
+          .try(:captures)
+          .try(:first)
+          .try(:to_f)
+      end
+    end
+  end
+end

data/lib/lakes/texas.rb

@@ -2,15 +2,26 @@ require 'net/http'
 require 'nokogiri'
 require 'date'
 require 'try'
+require 'lakes/helper'
 
 module Lakes
   class Texas
+    include Lakes::Helper
     attr_reader :lake_data
 
     def initialize
       @lake_data = {}
     end
 
+    def all_details
+      result = []
+      list.each do |lake_name|
+        result << get_details(lake_name)
+        sleep(1)
+      end
+      result
+    end
+
     def list
       return @lake_data.keys unless @lake_data.empty?
 
@@ -47,7 +58,7 @@ module Lakes
       main_div = html_doc.at('div#maincontent')
 
       parse_lake_characteristics(main_div, lake_data)
-      parse_water_conditions(main_div, lake_data)
+      parse_water_conditions_and_data(main_div, lake_data)
       parse_reservoir_controlling_authority(main_div, lake_data)
       parse_aquatic_vegetation(main_div, lake_data)
       parse_predominant_fish_species(main_div, lake_data)
@@ -122,8 +133,31 @@ module Lakes
       lake_data[:lake_characteristics][:year_impounded] = parser.year_impounded
     end
 
-    def parse_water_conditions(main_div, lake_data)
-      process_simple_section(main_div, lake_data, 'Water Conditions', :water_conditions, true)
+    def parse_water_conditions_and_data(main_div, lake_data)
+      lake_data[:raw_water_conditions] = process_simple_section(main_div, lake_data, 'Water Conditions', :water_conditions, true)
+
+      File.write("test/data/water_conditions/#{lake_data[:name]}.txt", lake_data[:raw_water_conditions])
+      parser = WaterConditionsParser.new(lake_data[:raw_water_conditions])
+      lake_data[:water] = {}
+      lake_data[:water][:conditions] = {}
+      lake_data[:water][:water_data_uri] = parser.water_data_uri
+      lake_data[:water][:conditions][:conservation_pool_elevation] = parser.conservation_pool_elevation
+      lake_data[:water][:conditions][:fluctuation] = parser.fluctuation
+      lake_data[:water][:conditions][:normal_clarity] = parser.normal_clarity
+
+      lake_data[:water][:data] = {}
+      return if parser.water_data_uri.nil?
+      content = begin
+        http_get(parser.water_data_uri)
+      rescue Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e
+        puts "#{e.message} for #{lake_data[:name]}: #{parser.water_data_uri}"
+        nil
+      end
+
+      return if content.nil?
+      water_data_parser = WaterDataParser.new(content)
+      lake_data[:water][:conservation_pool_elevation_in_ft_msl] = water_data_parser.conservation_pool_elevation_in_ft_msl
+      lake_data[:water][:percentage_full] = water_data_parser.conservation_pool_elevation_in_ft_msl
     end
 
     def parse_reservoir_controlling_authority(main_div, lake_data)
@@ -259,38 +293,11 @@ module Lakes
       data
     end
 
-    # converts this:
-    # ../../../action/waterecords.php?WB_code=0001
-    # into this:
-    # http://tpwd.texas.gov/fishboat/fish/action/waterecords.php?WB_code=0001
-    # based on this:
-    # http://tpwd.texas.gov/fishboat/fish/recreational/lakes/abilene
-    def convert_relative_href(href, current_url)
-      relative_depth = href.split('..').count - 1
-      url_parts = current_url.split('/')
-      url_parts.slice!(-relative_depth, relative_depth)
-      fixed_href = href.gsub('../', '')
-      url_parts.join('/') + '/' + fixed_href
-    end
-
-    # texas lake websites use lots of non breaking spaces
-    def cleanup_data(value)
-      nbsp = 160.chr('UTF-8')
-      value = value.strip.gsub(nbsp, '')
-      value.empty? ? nil : value
-    end
-
     def process_simple_section(main_div, lake_data, section_title, data_name, html)
       data = main_div.xpath("//h6[contains(text(), \"#{section_title}\")]").first
       element_type_function = html ? :to_html : :text
       content = data.try(:next_element).try(element_type_function)
       lake_data[data_name] = content
     end
-
-    # texas lake pages are encoded in Windows-1252 :(
-    def http_get(url)
-      uri = URI(url)
-      Net::HTTP.get(uri).encode('UTF-8', 'Windows-1252')
-    end
   end
 end

data/lib/lakes.rb

@@ -1,5 +1,7 @@
 require 'lakes/texas'
 require 'lakes/texas/lake_characteristics_parser'
+require 'lakes/texas/water_conditions_parser'
+require 'lakes/texas/water_data_parser'
 module Lakes
-  VERSION = '0.1.2'.freeze
+  VERSION = '0.1.3'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lakes
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Shane Sherman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-15 00:00:00.000000000 Z
+date: 2017-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -84,12 +84,15 @@ files:
 - lakes.gemspec
 - lakes.json
 - lib/lakes.rb
+- lib/lakes/helper.rb
 - lib/lakes/texas.rb
 - lib/lakes/texas/lake_characteristics_parser.rb
+- lib/lakes/texas/water_conditions_parser.rb
+- lib/lakes/texas/water_data_parser.rb
 - lib/try.rb
-homepage: 
+homepage: https://github.com/ssherman/lakes
 licenses:
-- mit
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []