smartermeter 0.3.3 → 0.4.0

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.4.0 (January 28, 2012)
+* Removed Google PowerMeter as it no longer exists
+* Rewrote PG&E scraper to work with OPower, their new web data provider.
+* Handle passwords greater than 8 characters.
+* Remove spec files from the gem.
+
 ## 0.3.3 (April 20, 2011)
 * Fixed a bug which prevent the proper Google PowerMeter authentication
   information to be sent.

data/README.md

@@ -6,8 +6,8 @@ really want to jump through 37 hoops to see the data on PG&E's website.
 So I made this.
 
 While making this library I discovered that PG&E doesn't even manage the
-software for the energy reporting. It's all done by energyguide.com. Not
-terribly useful but an interesting piece of trivia.
+software for the energy reporting. It's all done by OPower, not terribly
+useful but an interesting piece of trivia.
 
 Getting Started
 ---------------
@@ -34,25 +34,25 @@ to manipulate it using ruby like so:
 
 For futher information see the [API docs][rdoc]
 
-Google PowerMeter
+Pachube
 -----------------
 
 Once you've configured SmarterMeter once, you might want to use it with
-Google PowerMeter.
-
-1. Visit: https://www.google.com/powermeter/device/activate?mfg=Ruby&model=SmarterMeter&did=PGE&dvars=1
-1. Then sign in with your desired Google Account.
-1. Follow the directions on screen.
-1. On the final screen copy the entire "authInfo" into your favorite editor.
-   Pull out the "token" and the "path" from the string.
-1. Take the "path" you collected previously and append ".d1" to the end of it.
+Pachube, so you can visualize the results.
+
+1. Visit http://pachube.com and sign up for an account.
+1. Create a feed and a datastream.
+1. Copy the feed id (the last item in a feed url like 123 in
+   https://pachube.com/feeds/123) and the datastream id (which is the
+   name that you enter)
 1. Then append the following to your ~/.smartermeter file to
-   automatically upload data as it's retrieved from PG&E.
+automatically upload data as it's retrieved from PG&E.
 
-       :transport: :google_powermeter
-       :google_powermeter:
-         :token: "your-token"
-         :variable: "your-path-with.d1-appended"
+       :transport: :pachube
+       :pachube:
+         :api_key: "your-api-key"
+         :feed_id: "your-feed-id"
+         :datastream_id: "your-datastream-id"
 
 To Build the Windows Installer
 --------

data/Rakefile

@@ -43,24 +43,18 @@ end
 #
 #############################################################################
 
-task :default => :test
+task :default => :spec
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'spec/**/*_spec.rb'
-  test.verbose = true
-end
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
 
-desc "Generate RCov test coverage and open in your browser"
-task :coverage do
-  require 'rcov'
-  sh "rm -fr coverage"
-  sh "rcov test/test_*.rb"
-  sh "open coverage/index.html"
+desc "Generate code coverage"
+RSpec::Core::RakeTask.new(:coverage) do |t|
+  t.rcov = true
+  t.rcov_opts = ["--exclude", "spec"]
 end
 
-require 'rake/rdoctask'
+require 'rdoc/task'
 Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "#{name} #{version}"
@@ -221,6 +215,7 @@ task :gemspec => :validate do
     reject { |file| file =~ /^\./ }.
     reject { |file| file =~ /^(rdoc|pkg)/ }.
     reject { |file| file =~ /\.jar$/ }.
+    reject { |file| file =~ /^spec\// }.
     map { |file| "    #{file}" }.
     join("\n")
 

data/lib/smartermeter/daemon.rb

@@ -2,6 +2,7 @@ require 'fileutils'
 require 'crypt/blowfish'
 require 'yaml'
 require 'date'
+require 'base64'
 
 module SmarterMeter
   # @private
@@ -64,16 +65,14 @@ module SmarterMeter
 
     # Takes the unencrypted password and encrypts it.
     def password=(unencrypted)
-      padding_length = 8 - (unencrypted.bytesize % 8)
-      unencrypted = unencrypted + ("\0" * padding_length)
-      @config[:password] = cipher.encrypt_block(unencrypted)
+      @config[:password] = Base64.encode64(cipher.encrypt_string(unencrypted))
     end
 
     # Returns the clear-text password or nil if it isn't set.
     def password
-      password = @config.fetch(:password, nil)
+      password = Base64.decode64(@config.fetch(:password, nil))
       if password
-        cipher.decrypt_block(password).gsub("\0", "")
+        cipher.decrypt_string(password).gsub("\0", "")
       else
         password
       end
@@ -179,11 +178,11 @@ module SmarterMeter
         dates.each do |date|
           @ui.log.info("Fetching #{date}")
 
-          data = service.fetch_csv(date)
+          data = service.fetch_espi(date)
           next if data.empty?
 
           @ui.log.info("Verifying #{date}")
-          samples = Samples.parse_csv(data).values.first
+          samples = Samples.parse_espi(data).values.first
           first_sample = samples.first
 
           if first_sample.kwh
@@ -208,9 +207,9 @@ module SmarterMeter
 
     def upload(date, samples)
       case @config[:transport]
-      when :google_powermeter
-        @ui.log.info("Uploading #{date} to Google PowerMeter")
-        transport = SmarterMeter::Services::GooglePowerMeter.new(@config[:google_powermeter])
+      when :pachube
+        @ui.log.info("Uploading #{date} to Pachube")
+        transport = SmarterMeter::Services::Pachube.new(@config[:pachube])
         if transport.upload(samples)
           @ui.log.info("Upload for #{date} complete")
         else

data/lib/smartermeter/interfaces/swing.rb

@@ -187,11 +187,11 @@ module SmarterMeter
                   config = {
                     :username => @pages[0].username,
                     :password => @pages[0].password,
-                    :transport => :google_powermeter,
-                    :google_powermeter => {
-                      :token => @pages[1].token,
-                      :variable => @pages[1].variable,
-                      :auth => @pages[1].auth
+                    :transport => :pachube,
+                    :pachube => {
+                      :api_key => @pages[1].api_key,
+                      :feed_id => @pages[1].feed_id,
+                      :datastream_id => @pages[1].datastream_id
                     }
                   }
                 else
@@ -206,7 +206,7 @@ module SmarterMeter
           c.buttons = @buttons.build
 
           @panel = JPanel.new(CardLayout.new)
-          @pages = [PGEPage, GooglePowerMeterPage, CompletePage].map do |klass|
+          @pages = [PGEPage, PachubePage, CompletePage].map do |klass|
             page = klass.new(@buttons)
             @panel.add(page.build, klass.to_s)
             page
@@ -397,7 +397,7 @@ module SmarterMeter
         end
       end
 
-      class GooglePowerMeterPage
+      class PachubePage
         include_package "javax.swing"
         include_package "java.awt"
         include WizardPage
@@ -405,48 +405,51 @@ module SmarterMeter
         def initialize(buttons)
           @buttons = buttons
 
-          title = "<html><b>Connect to Google PowerMeter</b></html>"
-          message  = "In order to view your power data on Google PowerMeter you'll need to create an account."
+          title = "<html><b>Connect to Pachube</b></html>"
+          message  = "In order to view your power data on Pachube you'll need to create an account."
 
           header(title, message) do |c|
             layout = "
                 [ create ]
-                [ auth_label ]
-                [ auth_field ]
+                [ api_key_label | <api_key ]
+                [ feed_id_label | <feed_id ]
+                [ datastream_id_label | <datastream_id ]
             "
             @controls = Profligacy::Swing::LEL.new(JPanel, layout) do |cc,ii|
-              cc.create = JButton.new "Create a PowerMeter Account"
-              ii.create = { :action => method(:open_google_powermeter_registration) }
-              cc.auth_label = JLabel.new "Then, copy your Authorization Information below:"
-              cc.auth_field = JTextArea.new
-              cc.auth_field.line_wrap = true
-              cc.auth_field.minimum_size = Dimension.new(400, 80)
-              cc.auth_field.maximum_size = Dimension.new(400, 80)
-              ii.auth_field = { :key => method(:validate) }
+              cc.create = JButton.new "Create a Pachube Account"
+              ii.create = { :action => method(:open_pachube_registration) }
+              cc.api_key_label = JLabel.new "Api Key:"
+              cc.api_key_field = JTextField.new
+              ii.api_key_field = { :key => method(:validate) }
+              cc.feed_id_label = JLabel.new "Feed id:"
+              cc.feed_id_field = JTextField.new
+              ii.feed_id_field = { :key => method(:validate) }
+              cc.datastream_id_label = JLabel.new "Datastream Name:"
+              cc.datastream_id_field = JTextField.new
+              ii.datastream_id_field = { :key => method(:validate) }
             end
             c.controls = @controls.build
           end
         end
 
-        def token
-          CGI.parse(@controls.auth_field.text.strip)["token"][0]
+        def api_key
+          @controls.api_key.text.strip
         end
 
-        def variable
-          CGI.parse(@controls.auth_field.text.strip)["path"][0]+".d1"
+        def feed_id
+          @controls.feed_id.text.strip
         end
 
-        def auth
-          @controls.auth_field.text.strip
+        def datastream_id
+          @controls.datastream_id.text.strip
         end
 
-        # Opens the Google Powermeter registration flow so that users can upload
-        # their data.
+        # Opens the Pachube plans page so that users can register
         #
         # Returns nothing.
-        def open_google_powermeter_registration(*ignored_args)
+        def open_pachube_registration(*ignored_args)
             desktop = Desktop.getDesktop()
-            uri = Java::JavaNet::URI.new("https://www.google.com/powermeter/device/activate?mfg=SmarterMeter&model=SmarterMeter&did=PGE&dvars=1")
+            uri = Java::JavaNet::URI.new("https://pachube.com/plans")
             desktop.browse(uri)
         end
 
@@ -455,7 +458,9 @@ module SmarterMeter
         #
         # Returns nothing.
         def validate(*ignored_args)
-          @buttons.next.enabled = @controls.auth_field.text.size > 0
+          if api_key.any? && feed_id.any? && datastream_id.any?
+            @buttons.next.enabled = true
+          end
         end
       end
 

data/lib/smartermeter/samples.rb

@@ -1,44 +1,40 @@
 require 'date'
 require 'time'
+require 'nokogiri'
 
 module SmarterMeter
   # Represents a collection of samples. In some cases it's useful to operate on
   # groups of samples and this class provides that functionality.
   class Samples < Hash
-    # Parses the CSV returned by PG&E and creates a Samples collection.
+    # Parses the XML returned by PG&E and creates a Samples collection.
     #
-    # @param [String] data the string containing the CSV returned by PG&E
+    # @param [String] data the string containing the XML returned by PG&E
     # @return [Samples] creates a Samples collection from the given data.
-    def self.parse_csv(data)
+    def self.parse_espi(data)
       samples = Samples.new
-      date_re = /([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})/
 
-      # Apparently they felt the need to put a = outside of the correct place
-      data = data.gsub('=','')
+      doc = Nokogiri::HTML(data)
 
-      hour_increment = 60*60
-      CSV.parse(data) do |row|
-        next unless row.length > 0 and date_re.match row[0]
+      doc.xpath("//intervalreading").each do |reading|
+        # NOTE: This is a hack, the ESPI data seems to be assuming that
+        # all users live in the Eastern Time Zone. The timestamps
+        # returned in the ESPI should really be in UTC and not in local
+        # time. I'm going to assume all PG&E customers are in the
+        # pacific timezone and since the eastern timezone has the same
+        # daylight savings time rules then we can use a constant
+        # difference to correct the problem.
+        pacific_timezone_correction = 60*60*3
 
-        month, day, year = date_re.match(row[0]).captures
-        month = month.to_i
-        day = day.to_i
-        year = year.to_i
+        timestamp = Time.at(reading.xpath("./timeperiod/start").first.content.to_i + pacific_timezone_correction)
+        value = reading.xpath("./value").first.content.to_i / 900.0
 
-        timestamp = Time.local(year, month, day, 0) - hour_increment
-        next if row[1].include? "$"
-        hourly_samples = row[1..24].map do |v|
-          if v == "-"
-            kwh = nil
-          else
-            kwh = v.to_f
-          end
+        year = timestamp.year
+        month = timestamp.month
+        day = timestamp.day
 
-          timestamp = timestamp + hour_increment
-          Sample.new(timestamp, kwh)
-        end
-        samples[Date.new(year, month, day)] = hourly_samples
+        (samples[Date.new(year, month, day)] ||= []) << Sample.new(timestamp, value)
       end
+
       samples
     end
 

data/lib/smartermeter/service.rb

@@ -1,5 +1,8 @@
 require 'mechanize'
 require 'csv'
+require 'tempfile'
+require 'zip/zip'
+require 'uri'
 
 module SmarterMeter
   # Provides access to the PG&E SmartMeter data through a ruby interface. This
@@ -7,9 +10,11 @@ module SmarterMeter
   # so if something stops working its likely that something changed on PG&E's
   # site and this class will need to be adapted.
   class Service
-    LOGIN_URL = "http://www.pge.com/myhome/"
-    OVERVIEW_URL = "https://www.pge.com/csol/actions/login.do?aw"
-    ENERGYGUIDE_AUTH_URL = "https://www.energyguide.com/LoadAnalysis/LoadAnalysis.aspx?Referrerid=154"
+    LOGIN_FORM_URL = "http://www.pge.com/"
+    LOGIN_URL = "https://www.pge.com/eum/login"
+    MY_USAGE_URL = "https://www.pge.com/myenergyweb/appmanager/pge/customer?_nfpb=true&_pageLabel=MyUsage"
+    SAML_URL = "https://sso.opower.com/sp/ACS.saml2"
+    MY_ENERGY_USE_URL = "https://pge.opower.com/ei/app/myEnergyUse"
 
     # Provides access to the last page retrieved by mechanize. Useful for
     # debugging and reporting errors.
@@ -31,40 +36,15 @@ module SmarterMeter
     # @return [Boolean] true upon succesful login and false otherwise
     def login(username, password)
       begin
-        @agent.get(LOGIN_URL) do |page|
-          logged_in_page = page.form_with(:action => 'https://www.pge.com/eum/login') do |login|
+        @agent.get(LOGIN_FORM_URL) do |page|
+          saml_page = page.form_with(:action => LOGIN_URL) do |login|
             login.USER = username
             login.PASSWORD = password
+            login.TARGET = MY_USAGE_URL
           end.submit
-        end
-
-        # There is a crazy meta-reload thing here that mechanize doesn't handle
-        # correctly by itself so let's help it along...
-        @agent.get(OVERVIEW_URL) do |page|
-
-          return false if page.title =~ /PG&E Login/
-
-          # Load the PG&E Terms of Use page
-          tou_link = page.link_with(:href => '/csol/actions/billingDisclaimer.do?actionType=hourly')
-          unless tou_link
-            @last_page = page
-            return false
-          end
-          tou_page = @agent.click(tou_link)
-          form = tou_page.forms().first
-          agree_button = form.button_with(:value => 'I Understand - Proceed')
-
-          # Agree to the terms of use
-          form['agreement'] = 'yes'
-
-          # Load up the PG&E frame page for historical data
-          hourly_usage_container = form.submit(agree_button)
-
-          # Now load up the frame with the content
-          hourly_usage = @agent.click(hourly_usage_container.frames.select{|f| f.href == "/csol/nexus/content.jsp"}.first)
-
-          # Now post the authentication information from PG&E to energyguide.com
-          @data_page = hourly_usage.form_with(:action => ENERGYGUIDE_AUTH_URL).submit
+          token = saml_page.form_with(:action => SAML_URL).submit
+          usage_page = token.form_with(:action => MY_ENERGY_USE_URL).submit
+          @data_page = usage_page.link_with(:text => "Green Button").click
         end
         @authenticated = true
       rescue Exception => e
@@ -73,41 +53,44 @@ module SmarterMeter
       end
     end
 
-    # Downloads a CSV containing hourly date on that date. Up to a week worth
-    # of other data will be included depending on which day of the week that
-    # you request.
+    # Downloads an ESPI file containing data in 15 minute windows on the
+    # given date.
     #
-    # PG&E compiles this data on a weekly schedule so if you ask for Monday
-    # you'll get the previous Sunday and the following days upto the next
-    # Sunday.
-    #
-    # @return [String] the CSV data.
-    def fetch_csv(date)
-      raise RuntimeException, "login must be called before fetch_csv" unless @authenticated
+    # @param [Time] date - The day of the data you wish to fetch.
+    # @return [String] the XML data.
+    def fetch_espi(date)
+      raise RuntimeException, "login must be called before fetch_espi" unless @authenticated
 
-      # Now we almost actually have data. However we need to setup the desired
-      # parameters first before we can get the exportable data. This really shouldn't
-      # be necessary.
       begin
-        hourly_data = @data_page.form_with(:action => "LoadAnalysis.aspx") do |form|
-          form['__EVENTTARGET'] = "objChartSelect$butSubmit"
-          form['objTimePeriods$objExport$hidChart'] = "Hourly Usage"
-          form['objTimePeriods$objExport$hidChartID'] = 8
-          form['objChartSelect$ddChart'] = 8 # Hourly usage
-
-          form['objTimePeriods$objExport$hidTimePeriod'] = "Week"
-          form['objTimePeriods$objExport$hidTimePeriodID'] = 3
-          form['objTimePeriods$rlPeriod'] = 3
-
-          form['objChartSelect$ccSelectedDate1'] = date.strftime("%m/%d/%Y")
-        end.submit
-
-        # Now the beautiful data...
-        hourly_csv = hourly_data.form_with(:action => "LoadAnalysis.aspx") do |form|
-          form['__EVENTTARGET'] = "objTimePeriods$objExport$butExport"
-        end.submit
-
-        hourly_csv.body
+        form = @data_page.forms.first
+        begin
+          form.radiobutton_with(:value => 'ESPI_INTERVAL').click
+          form.field_with(:name => 'from').value = date.strftime("%m/%d/%Y")
+          form.field_with(:name => 'to').value = date.strftime("%m/%d/%Y")
+        end
+        espi_xml_zip = form.submit
+
+        # This has to be one of the stupidest implementations I've seen
+        # of a ruby library. Why on earth can you not pass in an IO
+        # object to the rubyzip library?
+        file = Tempfile.new('espi-zip')
+        begin
+          file.binmode
+          file << espi_xml_zip.body
+          file.flush
+          file.close
+
+          Zip::ZipInputStream::open(file.path) do |contents|
+            while (entry = contents.get_next_entry)
+              if (entry.name =~ /pge_electric_interval_data/) then
+                return contents.read
+              end
+            end
+          end
+        ensure
+          file.close
+          file.unlink
+        end
       rescue Exception => e
         @last_exception = e
         return ""

data/lib/smartermeter/services/pachube.erb

@@ -0,0 +1,2 @@
+<% samples.select { |s| s.kwh }.each do |sample| %>
+<%= sample.time.utc.iso8601 %>,<%= sample.kwh %><% end %>

data/lib/smartermeter/services/{google_powermeter.rb → pachube.rb}

@@ -1,13 +1,13 @@
 require 'net/https'
-require 'erb'
 
 module SmarterMeter
   module Services
-    class GooglePowerMeter
+    class Pachube
       def initialize(config)
         @config = config
-        raise "The Google PowerMeter token must be configured" unless @config[:token]
-        raise "The Google PowerMeter variable must be configured" unless @config[:variable]
+        raise "The Pachube token must be configured" unless @config[:api_key]
+        raise "The Pachube feed id must be configured" unless @config[:feed_id]
+        raise "The Pachube datastream id must be configured" unless @config[:datastream_id]
       end
 
       # Public: Uploads an array of Samples to Google PowerMeter
@@ -16,12 +16,12 @@ module SmarterMeter
       #
       # Returns true on success and false otherwise
       def upload(samples)
-        url = URI.parse('https://www.google.com/powermeter/feeds/event')
+        url = URI.parse("https://api.pachube.com/v2/feeds/#{@config[:feed_id]}/datastreams/#{@config[:datastream_id]}/datapoints.csv")
         http = Net::HTTP.new(url.host, url.port)
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_PEER
         http.ca_file = File.join(File.dirname(__FILE__), "cacert.pem")
-        res, body = http.post(url.path, request_body(samples), {"Authorization" => "AuthSub token=\"#{@config[:token]}\"", "Content-Type" => "application/atom+xml"})
+        res, body = http.post(url.path, request_body(samples), {"X-PachubeApiKey" => @config[:api_key], "Content-Type" => "text/csv"})
         case res
         when Net::HTTPSuccess
           true
@@ -30,12 +30,12 @@ module SmarterMeter
         end
       end
 
-      # Creates the proper XML request to send to Google.
+      # Creates the proper XML request to send to Pachube
       #
-      # Returns the proper atom/xml response to send to google
+      # Returns the proper text/csv request to send to pachube
       def request_body(samples)
-        template = ERB.new(File.read(File.join(File.dirname(__FILE__), "google_powermeter.erb")))
-        template.result(binding)
+        template = ERB.new(File.read(File.join(File.dirname(__FILE__), "pachube.erb")))
+        template.result(binding).gsub(/^\n/, '')
       end
     end
   end

data/lib/smartermeter.rb

@@ -2,10 +2,10 @@ require 'smartermeter/sample'
 require 'smartermeter/samples'
 require 'smartermeter/service'
 require 'smartermeter/daemon'
-require 'smartermeter/services/google_powermeter'
+require 'smartermeter/services/pachube'
 require 'smartermeter/services/brighterplanet'
 require 'smartermeter/interfaces/cli'
 
 module SmarterMeter
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

data/smartermeter.gemspec

@@ -13,8 +13,8 @@ Gem::Specification.new do |s|
   ## If your rubyforge_project name is different, then edit it and comment out
   ## the sub! line in the Rakefile
   s.name              = 'smartermeter'
-  s.version           = '0.3.3'
-  s.date              = '2011-04-21'
+  s.version           = '0.4.0'
+  s.date              = '2012-01-28'
   s.rubyforge_project = 'smartermeter'
 
   ## Make sure your summary is short. The description may be as long
@@ -52,12 +52,13 @@ Gem::Specification.new do |s|
   s.add_dependency('crypt19', ["= 1.2.1"])
   s.add_dependency('rest-client', ["= 1.6.1"])
   s.add_dependency('json_pure', ["= 1.5.1"])
+  s.add_dependency('rubyzip', ["= 0.9.5"])
 
   ## List your development dependencies here. Development dependencies are
   ## those that are only needed during development
-  s.add_development_dependency('rspec', ["~> 2.5.0"])
-  s.add_development_dependency('vcr', ["~> 1.7.0"])
-  s.add_development_dependency('webmock', ["~> 1.6.2"])
+  s.add_development_dependency('rspec', ["~> 2.8.0"])
+  s.add_development_dependency('vcr', ["~> 1.11.3"])
+  s.add_development_dependency('webmock', ["~> 1.7.10"])
   s.add_development_dependency('minitar', ["~> 0.5.2"])
 
   ## Leave this section as-is. It will be automatically generated from the
@@ -87,18 +88,9 @@ Gem::Specification.new do |s|
     lib/smartermeter/service.rb
     lib/smartermeter/services/brighterplanet.rb
     lib/smartermeter/services/cacert.pem
-    lib/smartermeter/services/google_powermeter.erb
-    lib/smartermeter/services/google_powermeter.rb
+    lib/smartermeter/services/pachube.erb
+    lib/smartermeter/services/pachube.rb
     smartermeter.gemspec
-    spec/fixtures/data.csv
-    spec/fixtures/expected_google_request.xml
-    spec/fixtures/vcr_cassettes/brighterplanet.yml
-    spec/sample_spec.rb
-    spec/samples_spec.rb
-    spec/service_spec.rb
-    spec/services/brighterplanet_spec.rb
-    spec/services/google_powermeter_spec.rb
-    spec/spec_helper.rb
   ]
   # = MANIFEST =