plantwatchdog 0.0.1

data/Manifest.txt

@@ -0,0 +1,42 @@
+config.ru
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/plantwatchdog
+bin/upload_measurements
+config/app_config.yaml
+lib/plantwatchdog/aggregation.rb
+lib/plantwatchdog/aggregation_methods.rb
+lib/plantwatchdog/data.rb
+lib/plantwatchdog/db.rb
+lib/plantwatchdog/gems.rb
+lib/plantwatchdog/main.rb
+lib/plantwatchdog/model.rb
+lib/plantwatchdog/sinatra.rb
+public/images/arrow-down.gif
+public/images/arrow-left.gif
+public/images/arrow-right.gif
+public/images/arrow-up.gif
+public/images/spinner.gif
+public/images/tabs.png
+public/js/customflot.js
+public/js/jquery-1.3.2.min.js
+public/js/jquery.flot.crosshair.js
+public/js/jquery.flot.js
+public/js/jquery.flot.navigate.js
+public/js/jquery.flot.selection.js
+public/js/select-chain.js
+public/js/tools.tabs-1.0.4.js
+public/tabs.css
+sample/solar/create_solar.rb
+sample/solar/measurements/client.sqlite3
+sample/solar/static/devices.yml
+sample/solar/static/metadata.yml
+sample/solar/static/plants.yml
+sample/solar/static/users.yml
+sample/solar/upload_measurements
+templates/graph.erb
+templates/index.erb
+templates/monthly_graph.erb

data/README.txt

@@ -0,0 +1,91 @@
+= Plant Watchdog
+
+A watchdog for your technical plant, e.g. a photovoltaic generator. The plant
+watchdog listens to data loggers, which continuously upload measurements of
+plant parameters. Based on that data the watchdog creates reports about the
+operational status.
+
+
+== DESCRIPTION:
+
+This software is being developed to monitor a photovoltaic generator. We found
+that it would make sense to keep the domain specific knowledge out of the code
+and instead provide a DSL to allow users to define their rules. Therefore this
+software should be useful to monitor any kind of technical plant. 
+
+At the time being the focus of development is to 
+- make it reasonably stable 
+- run per-day aggregations
+- show intraday and monthly diagrams
+
+With more data being available the task of finding deviations from regular 
+operation is becoming feasible. Therefore expected values of parameters must
+be calculated and compared with actual values: the users must be allowed to
+define a model of the plant.
+
+== FEATURES/PROBLEMS:
+
+The vision of this software is to
+ - allow data logger devices to continuously upload time series measurements
+ - backup and distribute measurement data 
+ - run user defined aggregations and analyses on the data
+ - provide customizable HTML reports including visual diagrams
+ - find deviations from regular operation and send alarms if necessary
+
+== REQUIREMENTS:
+
+* Ruby 1.8.7 and a database supported by active record, e.g. MySQL or SQLite
+
+== INSTALL:
+
+1. Install Ruby 1.8
+For ubuntu 
+  $ apt-get install ruby1.8-dev libopenssl-ruby1.8
+
+2. Install a database
+  Any database supported from active record should do, so far the server has
+  been tested with MySql 5.1.14 and SQLite 3.6.21.
+  
+  In order to install sqlite3 and ruby bindings on ubuntu run
+  $ apt-get install sqlite3-dev
+  $ gem install sqlite3-ruby  
+     
+3. Install Plant Watchdog and required gems
+  There are two options, either install the gem or download the sources from
+  github. 
+  A) gem 
+  $ gem install plantwatchdog  
+  
+  B) github
+  $ mkdir plant
+  $ cd plant
+  $ git clone git@github.com:mbarchfe/plantwatchdog.git
+  $ rake check_extra_deps   // Get the required ruby gems
+  
+4. Install sample database and run
+  If your gem's bin directory is not on the PATH, add it
+  $ export PATH=$PATH:/var/lib/gems/1.8/bin
+  
+  The default database connection uses sqlite3 and a database at 
+  /tmp/solarsample.sqlite3. You can change the default by editing
+  the config file
+  $ vi config/app-config.yaml
+  
+  Install the sample
+  $ plantwatch --create_sample
+  Upload data (needs curl and sqlite3 command line)
+  
+  Start the plantwatchdog web server (an alternative way is to start via rackup)
+  $ plantwatchdog
+  
+  On another shell upload sample measurements. The script needs curl and the
+  sqlite3 command line
+  $ apt-get install curl
+  $ upload_measurements
+ 
+  Run the daily aggregation
+  $ plantwatchdog -a
+
+== LICENSE:
+
+GPL v3

data/Rakefile

@@ -0,0 +1,22 @@
+# -*- ruby -*-
+$:.unshift File.join(File.dirname(__FILE__),"lib")
+require 'rubygems'
+require 'hoe'
+require 'plantwatchdog/main'
+
+Hoe.spec 'plantwatchdog' do |p|
+  developer('plantwatchdogteam', 'mbarchfe@rubyforge.org')
+  p.version=PlantWatchdog::Version::STRING
+  p.rubyforge_name = 'pwd'
+  p.author = "Plant Watchdog Team"
+  p.email = "mbarchfe@rubyforge.org"
+  p.summary = 'Plant Watchdog'
+  p.description = p.paragraphs_of('README.txt', 1..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps<<['sinatra','0.9.4']
+  p.extra_deps<<['activerecord', '2.3.5']
+  p.extra_deps<<['patir', '0.6.4']
+  p.spec_extras={:executables=>["plantwatchdog","upload_measurements"],
+    :default_executable=>"plantwatchdog"}
+end

data/bin/plantwatchdog

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# an alternative way of starting the server is via rackup:
+# 1. bin$ gem install rack
+# 2. bin$ cd .. 
+# 3. $ rackup
+#d=`dirname $0`
+#ruby -I${d:-.}/../lib -rubygems ${d:-.}/../lib/plantwatchdog/main.rb $*
+require 'plantwatchdog/main'

data/bin/upload_measurements

@@ -0,0 +1,2 @@
+#!/usr/bin/env ruby
+exec File.join(File.dirname(__FILE__),"..", "sample", "solar", "upload_measurements")

data/config.ru

@@ -0,0 +1,35 @@
+# this is Rackup file. It is necessary for applications started from a container
+# e.g. passenger, or via the rackup command.
+require 'rubygems'
+#this locks on the versions we need
+require 'lib/plantwatchdog/gems'
+require 'lib/plantwatchdog/sinatra.rb'
+require 'lib/plantwatchdog/db.rb'
+require 'yaml'
+path = ''
+
+set :root, path
+set :views, path + '/views'
+set :public,  path + '/public'
+set :run, false
+set :raise_errors, true
+
+log = File.new("sinatra.log", "a")
+STDOUT.reopen(log)
+STDERR.reopen(log)
+
+logger=Logger.new(STDOUT)
+logger.level = Logger::INFO
+
+config_file="config/app_config.yaml"
+if File.exists?(config_file)
+      config=YAML.load(File.read(config_file))
+      config[:logger]=logger
+else
+      logger.fatal("Cannot find #{config_file}")
+      exit 1
+end
+
+extend PlantWatchdog::ActiveRecordConnections
+self.connect_to_active_record(config[:database_configuration],logger)
+run PlantWatchdog::UI::SinatraApp

data/config/app_config.yaml

@@ -0,0 +1,10 @@
+#---
+#:database_configuration: 
+#    :adapter: mysql
+#    :database: solar
+#    :username: root
+#    :password: root
+
+:database_configuration: 
+    :adapter: sqlite3
+    :database: /tmp/solarsample.sqlite3

data/lib/plantwatchdog/aggregation.rb

@@ -0,0 +1,220 @@
+# Copyright (C) 2010 Markus Barchfeld, Vassilis Rizopoulos
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+
+$:.unshift File.join(File.dirname(__FILE__),"..")
+require 'plantwatchdog/model'
+require 'plantwatchdog/aggregation_methods'
+
+module PlantWatchdog
+  # define the Aggregation Model,
+  module Aggregation
+    # the global environment to which the aggregation blocks have access
+    class AggregationEnv
+      logger = ActiveRecord::Base.logger
+      attr_accessor :year, :day_of_year, :plant, :devices
+      def initialize(year, day_of_year)
+        @year = year
+        @day_of_year = day_of_year
+        @devices = []
+      end
+
+    end
+
+    class Data
+      # return the time series data for symbol
+      def [] symbol
+
+      end
+
+      # return the list of seconds of the day for which there is data available
+      def times
+      end
+    end
+
+    module RuleEvaluation
+      # data is an array of dictionaries, e.g. retrieved via Measurements.from_csv
+      def eval_rule rule_array, data
+        return 0 if rule_array.empty?
+        method_name = rule_array.first
+        arg_desc = rule_array[1, rule_array.size] # the column names for which we need to create time series
+        args = arg_desc.collect { |arg_desc|
+          if arg_desc.is_a? Numeric then
+            arg_desc
+          elsif arg_desc.is_a? Array
+            eval_rule(arg_desc, data)
+          else
+            data.collect { |d| d[arg_desc] }
+          end
+        }
+        return Methods.call(method_name, *args)
+      end
+    end
+
+    class Device
+      include RuleEvaluation
+      def Device.create(model_device, year, day)
+        # select data
+        data = Model::MeasurementChunk.find(:first, :conditions => ["device_id=? and time_year=? and time_day_of_year=?", model_device.id, year, day])
+        Device.new(model_device, data)
+      end
+      attr_accessor :model_device
+
+      def initialize(model_device, data)
+        @model_device = model_device
+        @aggregates = {}
+        @data = data
+      end
+
+      # generic access to the fields of the underlying model_device
+      def not_understand
+        # model_device
+      end
+
+      def measurements
+        @data ? @data.measurements : []
+      end
+
+      def meta
+        model_device.meta
+      end
+
+      # return the dict with aggregated values
+      def aggregates
+
+      end
+
+      # execute the aggregation rules of the device
+      def aggregate
+        result = {}
+        logger.debug "Aggregating device #{model_device.id}, aggrules: #{model_device.aggrules}"
+        model_device.aggrules.each_pair do
+          |agg_key, rule_array|
+          result[agg_key] = eval_rule(rule_array, measurements)
+        end
+        logger.debug "Aggregation results: " + result.to_s
+        result
+      end
+
+      def persist
+        dm = Model::DailyMeasurement.new()
+        dm.description = JSON(aggregates)
+        return dm
+      end
+
+      # TODO: better way to access logger
+      def logger
+        return ActiveRecord::Base.logger
+      end
+    end
+
+    class Plant
+      include RuleEvaluation
+      def initialize model_plant, device_aggregates
+        @model_plant = model_plant
+        @device_aggregates = device_aggregates
+      end
+
+      def aggregate
+        result = {}
+        logger.debug "Aggregating plant, aggrules: #{@model_plant.aggrules}"
+        @model_plant.aggrules.each_pair do
+          |agg_key, rule_array|
+          result[agg_key] = eval_rule(rule_array,  @device_aggregates)
+        end
+        logger.debug "Aggregation results: " + result.to_s
+        result
+      end
+
+      # TODO: better way to access logger
+      def logger
+        return ActiveRecord::Base.logger
+      end
+
+    end
+
+    class Runner
+      def find_missing_aggregates
+        sql = <<EOF
+      select time_year, time_day_of_year, plant_id from
+      (select CHUNK.time_year, CHUNK.time_day_of_year, CHUNK.device_id AS device_id, AGG.device_id AS agg_device_id from
+      ((select time_year, time_day_of_year, device_id from measurement_chunks where type="MeasurementChunk" order by time_year, time_day_of_year, device_id) AS CHUNK 
+      LEFT OUTER JOIN
+      (select time_year, time_day_of_year, device_id from measurement_chunks where type="MeasurementAggregate") AS AGG 
+      ON CHUNK.time_year = AGG.time_year AND CHUNK.time_day_of_year=AGG.time_day_of_year AND CHUNK.device_id = AGG.device_id)
+      where AGG.device_id IS NULL) AS MISSING
+      INNER JOIN
+      devices
+      ON MISSING.device_id = devices.id
+      GROUP BY MISSING.time_year, MISSING.time_day_of_year;    
+EOF
+        rows = ActiveRecord::Base.connection.select_all(sql)
+        result = []
+        rows.collect {
+          |r|
+          time_year = r["time_year"].to_i
+          time_day_of_year = r["time_day_of_year"].to_i
+          plant_id = r["plant_id"].to_i
+          [time_year, time_day_of_year, plant_id]
+        }
+      end
+
+      def run
+        find_missing_aggregates.each {
+          |m|
+          time_year, time_day_of_year, plant_id = m
+          aggregate(Model::Plant.find_by_id(plant_id), time_year, time_day_of_year)
+        }
+      end
+
+      def aggregate(model_plant, year, day_of_year)
+        env = AggregationEnv.new(year, day_of_year)
+        model_plant.devices.each {
+          |model_device|
+          logger.debug("Adding device " + model_device.to_s)
+          env.devices << Device.create(model_device, year, day_of_year)
+        }
+        # build the aggregates for the devices first ...
+        aggregates = env.devices.collect do
+          |device|
+          daily = Model::MeasurementAggregate.new
+          daily.device = device.model_device
+          daily.time_year = year
+          daily.time_day_of_year = day_of_year
+          daily.data = device.aggregate
+          daily
+        end
+
+        # ... and then aggregate the plant
+        gen_aggregates = Plant.new(model_plant, aggregates.collect{|ma| ma.data}).aggregate
+
+        gen_aggregate = Model::MeasurementAggregate.new
+        gen_aggregate.data = gen_aggregates
+        gen_aggregate.time_year = year
+        gen_aggregate.time_day_of_year = day_of_year
+        gen_aggregate.plant = model_plant
+
+        # save when everything has been calculated
+        aggregates << gen_aggregate
+        aggregates.each {|a| a.save}
+        aggregates
+      end
+
+      # TODO: better way to access logger
+      def logger
+        return ActiveRecord::Base.logger
+      end
+
+    end
+  end
+end

data/lib/plantwatchdog/aggregation_methods.rb

@@ -0,0 +1,90 @@
+# Copyright (C) 2010 Markus Barchfeld, Vassilis Rizopoulos
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+
+$:.unshift File.join(File.dirname(__FILE__),"..")
+require 'plantwatchdog/model'
+
+class Array
+  def each_prior()
+    i=0
+    result=[]
+    while (i < self.size-1) do
+      result << yield(self[i],self[i+1])
+      i+=1
+    end
+    return result
+  end
+end
+
+module PlantWatchdog
+  module Aggregation
+    module Methods
+      class << self
+        def growth timeseries
+          timeseries.last - timeseries.first
+        end
+
+        def avg timeseries
+          result = sum(timeseries)
+          result.to_f / timeseries.size
+        end
+
+        def integrate times, values
+          return [times.each_prior {|x,y| y-x}, values.each_prior {|x,y| (y+x)/2.0}].transpose.inject(0) {|i,a| i + a.first * a.last }
+        end
+
+        def sum timeseries
+          result = 0
+          timeseries.each { |v| result += v if v}
+          return result
+        end
+
+        def mult a,b
+          a*b
+        end
+
+        def div a,b
+          a/b
+        end
+
+        def subtract a,b
+          a - b
+        end
+
+        def add
+          a + b
+        end
+
+        def pick n,a
+          a[n]
+        end
+
+        def call(method, *args)
+          begin
+            m = Methods.method method
+            m.call *args
+          rescue
+            logger.debug("Error calling method '#{method}': " + $!.to_s)
+            nil
+          end
+        end
+
+        def logger
+          return ActiveRecord::Base.logger
+        end
+      end
+
+    end
+  end
+end