smartermeter 0.1.0

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/README.md

@@ -0,0 +1,65 @@
+Smartermeter - the smarter way to read your PGE SmartMeter
+=========================================================
+
+So I have PGE SmartMeter and I like playing with data. However I didn't really
+want to jump through 37 hoops to see the data on PG&E's website. So I made
+this.
+
+While making this library I discovered that PG&E doesn't even manage the
+software for the energy reporting. It's all done by energyguide.com. Not
+terribly useful but an interesting piece of trivia.
+
+Getting Started
+---------------
+
+    gem install smartermeter
+    smartermeter
+
+Google PowerMeter
+-----------------
+
+Once you've configured smartermeter once, you might want to use it with Google
+PowerMeter.
+
+1. Visit: https://www.google.com/powermeter/device/activate?mfg=Ruby&model=SmarterMeter&did=PGE&dvars=1
+1. Then sign in with your desired Google Account.
+1. Follow the directions on screen.
+1. On the final screen copy the entire "authInfo" into your favorite editor.
+   Pull out the "token" and the "path" from the string.
+1. Take the "path" you collected previously and append ".d1" to the end of it.
+1. Then append the following to your ~/.smartermeter file to
+   automatically upload data as it's retrieved from PG&E.
+
+       :transport: :google_powermeter
+       :google_powermeter:
+         :token: "your-token"
+         :variable: "your-path-with.d1-appended"
+
+To Build
+--------
+
+In order to build the self contained binaries, you'll need a working jruby interpreter.
+
+    git clone git://github.com/mcolyer/smartermeter.git
+    cd smartermeter
+    rake rawr:prepare
+    rake rawr:bundle:exe # builds a windows executable
+    rake rawr:bundle:app # builds an OSX executable
+    rake build # builds a ruby gem
+
+Questions
+---------
+
+* How much lag is there?
+
+  It'll show you the last full day's worth of data. The PGE website claims that
+  data becomes available around 3-10pm on the following day. However my
+  experience says that it's sometimes available earlier.
+
+* How long is data saved for?
+
+  I don't know. If you know tell me.
+
+* How can I help?
+
+  Make sure it works, make cool things with it or send me git pull requests.

data/Rakefile

@@ -0,0 +1,182 @@
+require 'rubygems'
+require 'rake'
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def rubyforge_project
+  name
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'specs/*_spec.rb'
+  test.verbose = true
+end
+
+desc "Generate RCov test coverage and open in your browser"
+task :coverage do
+  require 'rcov'
+  sh "rm -fr coverage"
+  sh "rcov test/test_*.rb"
+  sh "open coverage/index.html"
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "#{name} #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+require 'rawr'
+
+namespace :rawr do
+  task :prepare do
+    dir = File.join(File.dirname(__FILE__), "vendor", "gems")
+    FileUtils.rm_rf(dir)
+    FileUtils.mkdir_p(dir)
+    ["nokogiri", "mechanize", "crypt"].each do |gem|
+      `gem unpack -t "#{dir}" #{gem}`
+    end
+
+    # Rawr can't handle folders with dashes in the name, so we'll remove the
+    # version numbers from the gems.
+    Dir.glob(File.join(dir, "*-*")).each do |gem|
+      no_version = File.basename(gem).split("-")[0] + ".old"
+      FileUtils.mv(gem, File.join(dir, no_version))
+    end
+
+    Dir.glob(File.join(dir, "nokogiri.old", "lib", "*")).each do |file|
+      FileUtils.mv(file, File.join(dir))
+    end
+    FileUtils.mv(File.join(dir, "crypt.old", "crypt"), File.join(dir, "crypt"))
+    Dir.glob(File.join(dir, "mechanize.old", "lib", "*")).each do |file|
+      FileUtils.mv(file, File.join(dir))
+    end
+
+    Dir.glob(File.join(dir, "*.old")).each do |gem|
+      FileUtils.rm_rf(gem)
+    end
+  end
+end
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+desc "Create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push origin master"
+  sh "git push origin v#{version}"
+  #sh "gem push pkg/#{name}-#{version}.gem"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec => :validate do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    reject { |file| file =~ /^rawr/ }.
+    reject { |file| file =~ /\.jar$/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+
+desc "Validate #{gemspec_file}"
+task :validate do
+  libfiles = Dir['lib/*'] - ["lib/#{name}.rb", "lib/#{name}"]
+  unless libfiles.empty?
+    puts "Directory `lib` should only contain a `#{name}.rb` file and `#{name}` dir."
+    exit!
+  end
+  unless Dir['VERSION*'].empty?
+    puts "A `VERSION` file at root level violates Gem best practices."
+    exit!
+  end
+end

data/bin/smartermeter

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+begin
+  require 'jruby'
+  # This is needed for nokogiri to function properly under jruby
+  JRuby.objectspace=true
+rescue Exception
+end
+require 'smartermeter'
+
+SmarterMeter::Daemon.new.start

data/build_configuration.rb

@@ -0,0 +1,89 @@
+configuration do |c|
+  # The name for your resulting application file (e.g., if the project_name is 'foo' then you'll get foo.jar, foo.exe, etc.)
+  # default value: "smartermeter"
+  #
+  #c.project_name = "smartermeter"
+
+  # Undocumented option 'output_dir'
+  # default value: "package"
+  #
+  c.output_dir = "pkg"
+
+  # The main ruby file to invoke, minus the .rb extension
+  # default value: "main"
+  #
+  c.main_ruby_file = "smartermeter/main"
+
+  # The fully-qualified name of the main Java file used to initiate the application.
+  # default value: "org.rubyforge.rawr.Main"
+  #
+  #c.main_java_file = "org.rubyforge.rawr.Main"
+
+  # A list of directories where source files reside
+  # default value: ["src"]
+  #
+  c.source_dirs = ["lib", "vendor/gems", "rawr"]
+
+  # A list of regexps of files to exclude
+  # default value: []
+  #
+  #c.source_exclude_filter = []
+
+  # Whether Ruby source files should be compiled into .class files
+  # default value: true
+  #
+  #c.compile_ruby_files = true
+
+  # A list of individual Java library files to include.
+  # default value: []
+  #
+  #c.java_lib_files = []
+
+  # A list of directories for rawr to include . All files in the given directories get bundled up.
+  # default value: ["lib/java"]
+  #
+  c.java_lib_dirs = ["vendor/java"]
+
+  # Undocumented option 'files_to_copy'
+  # default value: []
+  #
+  #c.files_to_copy = []
+
+  # Undocumented option 'target_jvm_version'
+  # default value: 1.6
+  #
+  #c.target_jvm_version = 1.6
+
+  # Undocumented option 'jvm_arguments'
+  # default value: ""
+  #
+  #c.jvm_arguments = ""
+
+  # Undocumented option 'java_library_path'
+  # default value: ""
+  #
+  #c.java_library_path = ""
+
+  # Undocumented option 'extra_user_jars'
+  # default value: {}
+  #
+  #c.extra_user_jars[:data] = { :directory => 'data/images/png',
+  #                             :location_in_jar => 'images',
+  #                             :exclude => /*.bak$/ }
+
+  # Undocumented option 'mac_do_not_generate_plist'
+  # default value: nil
+  #
+  #c.mac_do_not_generate_plist = nil
+
+  # Undocumented option 'mac_icon_path'
+  # default value: nil
+  #
+  #c.mac_icon_path = nil
+
+  # Undocumented option 'windows_icon_path'
+  # default value: nil
+  #
+  #c.windows_icon_path = nil
+
+end

data/lib/smartermeter/daemon.rb

@@ -0,0 +1,236 @@
+require 'crypt/blowfish'
+require 'yaml'
+require 'logger'
+require 'date'
+
+module SmarterMeter
+  class Daemon
+
+    # Loads the configuration, and starts
+    #
+    # Never returns.
+    def start
+      configure
+      run
+    end
+
+  protected
+    def config_file
+      File.expand_path("~/.smartermeter")
+    end
+
+    def default_data_dir
+      File.expand_path(File.join(File.dirname(__FILE__), "..", "data"))
+    end
+
+    # Returns a filename for the data belonging to the given date.
+    def data_file(date)
+      File.expand_path(File.join(@config[:data_dir], date.strftime("%Y-%m-%d.csv")))
+    end
+
+    def log
+      return @logger if @logger
+      @logger = Logger.new STDOUT
+      @logger.level = Logger::INFO
+      @logger
+    end
+
+    # Loads the configuration and prompts for required settings if they are
+    # missing.
+    #
+    # Returns nothing.
+    def configure
+      load_configuration
+      verify_configuration
+    end
+
+    # Loads the configuration from disk.
+    #
+    # Returns the configuration hash.
+    def load_configuration
+      @config = {
+        :start_date => Date.today,
+        :data_dir => default_data_dir
+      }
+
+      if File.exist?(config_file)
+        @config = YAML.load_file(config_file)
+      end
+
+      @config
+    end
+
+    def cipher
+      Crypt::Blowfish.new("Our easily discoverable key.")
+    end
+
+    # Takes the unencrypted password and encrypts it.
+    def password=(unencrypted)
+      @config[:password] = cipher.encrypt_block(unencrypted)
+    end
+
+    # Returns the clear-text password or nil if it isn't set.
+    def password
+      password = @config.fetch(:password, nil)
+      if password
+        cipher.decrypt_block(password)
+      else
+        password
+      end
+    end
+
+    # Prompts the user for required settings that are blank.
+    #
+    # Returns nothing.
+    def verify_configuration
+      return if @config[:username] and @config[:password]
+
+      puts
+      puts "Smartermeter: Initial Configuration"
+      puts "--------------------------------------------------------------------------------"
+      puts "This program stores your PG&E account username and password on disk. The"
+      puts "password is encrypted but could be retrieved fairly easily. If this makes you"
+      puts "uncomfortable quit now (use ctrl-c)."
+      puts "--------------------------------------------------------------------------------"
+
+      unless @config[:username]
+        print "PG&E account username: "
+        @config[:username] = gets.strip
+      end
+
+      unless @config[:password]
+        print "PG&E account password: "
+        self.password = gets.strip
+      end
+
+      save_configuration
+      puts "Setup complete"
+    end
+
+    # Saves the current configuration to disk.
+    #
+    # Returns nothing.
+    def save_configuration
+      File.open(config_file, "w") do |file|
+        file.write(YAML.dump(@config))
+      end
+    end
+
+    # Continually checks for new data for any missing days, since the first day
+    # smartermeter started watching.
+    #
+    # Never returns.
+    def run
+      one_hour = 60 * 60
+
+      while true
+        dates = dates_requiring_data
+        unless dates.empty?
+          log.info("Attempting to fetch data for: #{dates.join(",")}")
+          results = fetch_dates(dates)
+          log.info("Successfully fetched: #{results.join(",")}")
+        else
+          log.info("Sleeping")
+        end
+        sleep(one_hour)
+      end
+    end
+
+    # Create an authorized Service instance.
+    #
+    # Note: An authorization failure will cause an exits, as it is a dire
+    # condition.
+    #
+    # Returns a new Service instance which has been properly authorized.
+    def service
+      service = Service.new
+      log.info("Logging in as #{@config[:username]}")
+      unless service.login(@config[:username], password)
+        log.error("Incorrect username or password given.")
+        log.error("Please remove ~/.smartermeter and configure smartermeter again.")
+        exit(-1)
+      end
+      log.info("Logged in as #{@config[:username]}")
+      service
+    end
+
+    # Connect and authenticate to the PG&E Website.
+    #
+    # It provides an instance of Service to the provided block
+    # for direct manipulation.
+    #
+    # Returns nothing.
+    def connect
+      s = service
+      begin
+        yield s
+      rescue SocketError => e
+        log.error("Could not access the PG&E site, are you connected to the Internet?")
+      end
+    end
+
+    # Attempts to retrieve power data for each of the dates in the list.
+    #
+    # dates - An array of Date objects to retrieve power data for.
+    #
+    # Returns an Array of successfully retrieved dates.
+    def fetch_dates(dates)
+      completed = []
+
+      connect do |service|
+        dates.each do |date|
+          log.info("Fetching #{date}")
+          data = service.fetch_csv(date)
+
+          log.info("Verifying #{date}")
+          samples = Sample.parse_csv(data).values.first
+          first_sample = samples.first
+
+          if first_sample.kwh
+            log.info("Saving #{date}")
+            File.open(data_file(date), "w") do |f|
+              f.write(data)
+            end
+
+            upload(date, samples)
+
+            log.info("Completed #{date}")
+            completed << date
+          else
+            log.info("Incomplete #{date}")
+          end
+        end
+      end
+
+      completed
+    end
+
+    def upload(date, samples)
+      case @config[:transport]
+      when :google_powermeter
+        log.info("Uploading #{date} to Google PowerMeter")
+        transport = SmarterMeter::Transports::GooglePowerMeter.new(@config[:google_powermeter])
+        if transport.upload(samples)
+          log.info("Upload for #{date} complete")
+        else
+          log.info("Upload for #{date} failed")
+        end
+      end
+    end
+
+    # Returns an Array of Date objects containing all dates since start_date
+    # missing power data.
+    def dates_requiring_data
+      collected = Dir.glob(File.join(@config[:data_dir], "*-*-*.csv")).map { |f| File.basename(f, ".csv") }
+      all_days = []
+
+      count_of_days = (Date.today - @config[:start_date]).to_i
+
+      count_of_days.times do |i|
+        all_days << (@config[:start_date] + i).strftime("%Y-%m-%d")
+      end
+
+      (all_days - collected).map { |d| Date.parse(d) }
+    end
+  end
+end

data/lib/smartermeter/main.rb

@@ -0,0 +1,9 @@
+# Used only to launch from a jar using rawr
+
+require 'jruby'
+# This is needed for nokogiri to function properly under jruby
+JRuby.objectspace=true
+
+require 'smartermeter'
+
+SmarterMeter::Daemon.new.start

data/lib/smartermeter/sample.rb

@@ -0,0 +1,68 @@
+require 'time'
+
+module SmarterMeter
+  class Sample
+    attr_accessor :time, :kwh
+
+    # Parses the CSV returned by PG&E
+    #
+    # data - The string containing the CSV returned by PG&E
+    #
+    # Returns a Hash of with keys as Date objects and values of Arrays of samples.
+    def self.parse_csv(data)
+      samples = {}
+      date_re = /([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})/
+
+      # Apparently they felt the need to put a = outside of the correct place
+      data = data.gsub('=','')
+
+      hour_increment = 1/24.0
+      CSV.parse(data) do |row|
+        next unless row.length > 0 and date_re.match row[0]
+
+        month, day, year = date_re.match(row[0]).captures
+        month = month.to_i
+        day = day.to_i
+        year = year.to_i
+
+        timestamp = Time.local(year, month, day, 0) - hour_increment + 1/(24.0*60)
+        hourly_samples = row[1..24].map do |v|
+          if v == "-"
+            kwh = nil
+          else
+            kwh = v.to_f
+          end
+
+          timestamp = timestamp + hour_increment
+          Sample.new(timestamp, kwh)
+        end
+        samples[Date.new(year, month, day)] = hourly_samples
+      end
+      samples
+    end
+
+    def initialize(time, kwh)
+      @time = time
+      @kwh = kwh
+    end
+
+    # Public: The start time of this measurement
+    #
+    # Returns the time in the format of 2011-01-06T09:00:00.000Z
+    def utc_start_time
+      @time.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+    end
+
+    # Public: The stop time of this measurement, it's assumed to be 1 hour after
+    # the start.
+    #
+    # Returns the time in the format of 2011-01-06T09:00:00.000Z
+    def utc_stop_time
+      (@time + 60*60).utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+    end
+
+    def inspect
+      "<Sample #{@time} #{@kwh}>"
+    end
+  end
+end