lewt 0.5.12

data/lib/extensions/calendar-timekeeping/calendar-timekeeping.rb

@@ -0,0 +1,65 @@
+load File.expand_path('../extractor.rb', __FILE__)
+load File.expand_path('../gcal_extractor.rb', __FILE__)
+load File.expand_path('../ical_extractor.rb', __FILE__)
+load File.expand_path('../apple_extractor.rb', __FILE__)
+
+
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+
+module LEWT
+
+  # The Calander Timekeeping LEWT Extensions lets you extract timesheet data from your iCal, Google Calender, or OSX Calender
+  # sources. In the process it transforms the data into a LEWTBook ready for processing.
+  # 
+  # In order for your calender events to be recognised as billable timesheet entries there are some naming conventions you must
+  # observe.
+  #
+  # ===Conventions:
+  # - The <tt>title</tt> of your event must contain a client name or alias reference in it.
+  # - The <tt>description</tt> of your event will be pulled into the LEWTBook description column.
+  include CalendarExtractors
+
+  class CalendarTimekeeping < LEWT::Extension
+    
+    # Sets up this extension and registers its options.
+    def initialize
+      # set extension options
+      options = {
+        :calendar => {
+          :default => "ical",
+          :definition => "The calender extraction method to use, supports 'gcal', 'ical', 'osx' calender extraction. Defaults to ical.",
+          :type => String,
+        },
+        :suppress => {
+          :definition => "Suppresses the cost calculation for the specified targets when calulating the hourly rates on extracted calender data.",
+          :type => String
+        }
+      }    
+      super({:cmd => "calendar", :options => options})
+    end
+
+    # Extracts data from a given calender source based on what was passed in the <tt>options['ext_method']</tt> parameter.
+    # options [Hash]:: The options hash passed to this function by the Lewt program.
+    # returns:: LEWTBook
+    def extract( options )
+      targetCustomers = self.get_matched_customers( options[:target], options[:suppress] )
+      dStart =  options[:start]
+      dEnd = options[:end]
+      suppressTargets = options[:suppress] == nil ? nil : self.get_matched_customers(options[:suppress])
+      if options[:calendar] == "ical"
+        extract = CalendarExtractors::ICalExtractor.new( dStart, dEnd, targetCustomers, lewt_settings, suppressTargets )
+      elsif options[:calendar] == "gcal"
+        extract = CalendarExtractors::GCalExtractor.new(dStart, dEnd, targetCustomers, lewt_settings, suppressTargets )
+      elsif options[:calendar] == "osx"
+        extract = CalendarExtractors::AppleExtractor.new(dStart, dEnd, targetCustomers, lewt_settings, suppressTargets )
+      end
+      return extract.data
+    end
+
+  end
+
+end

data/lib/extensions/calendar-timekeeping/extractor.rb

@@ -0,0 +1,62 @@
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+
+module CalendarExtractors
+
+  # The CalExtractor class acts as base class for various calender extraction interfaces such as GCalExtractor, ICalExtractor
+  # AppleExtractor. It provides some convenience methods that are useful across the various implementations.
+  class CalExtractor < LEWT::Extension
+    
+    attr_reader :data
+
+    # Initialises this class. This method should be invoked by sub-classes with <tt>super()</tt>. It invokes
+    # <tt>extractCalenderData</tt> on the sub-classes behalf when called...
+    # dateStart [String]:: a human readable date as a string for the start time period
+    # dateEnd [String]:: a human readable date as a string for the end time period
+    # targets [Hash]:: a hash containing all the targets returned by the LewtExtension.get_matched_customers() method
+    def initialize( dateStart, dateEnd, targets )
+      @data = LEWT::LEWTBook.new
+      @dateStart  = DateTime.parse dateStart.to_s 
+      @dateEnd = DateTime.parse dateEnd.to_s
+      @targets = targets
+      @category = "Hourly Income"
+      self.extractCalendarData
+    end
+    
+    # Returns the extracted calendar data. Must be implimented by subclasses.
+    def extractCalendarData
+      
+    end
+    
+    # Matches a search string against customer names/aliases
+    # evtSearch [String]:: a string to search against such as the title of an event
+    # returns:: false when no match found or the target customer details (as a hash) when matched
+    def isTargetCustomer? ( evtSearch )
+      match = false
+      @targets.each do |t|
+        reg = [ t['alias'], t['name'] ]
+        regex = Regexp.new( reg.join("|"), Regexp::IGNORECASE )
+        match = regex.match(evtSearch) != nil ? t : false;
+        break if match != false
+      end
+      return match
+    end
+    
+    # Checks whether an event date is within target range
+    # date [Date]:: the date to check against
+    # return:: Boolean true/false operation status
+    def isTargetDate? ( date ) 
+      d = DateTime.parse(date.to_s)
+      check = false
+      if d >= @dateStart && d <= @dateEnd
+        check = true
+      end
+      return check
+    end
+
+  end
+
+end

data/lib/extensions/calendar-timekeeping/gcal_extractor.rb

@@ -0,0 +1,61 @@
+require 'google_calendar'
+
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+module CalendarExtractors
+
+  # Extracts data from a Google Calender source.
+  #
+  # === Setup
+  # It is required that you have a Google App setup with your account to allow for API access to your data. Then 
+  # add the following keys to your settings file.
+  #
+  # gmail_username:: The username for your google application.
+  # gmail_password:: The password associated with this username.
+  # gmail_app_name:: The name of the application you created.
+  # 
+  class GCalExtractor < CalExtractor
+
+    # Sets up this extension
+    def initialize ( dStart, dEnd, targetCustomers, lewt_settings, suppressTargets )
+      uname = lewt_settings["gmail_username"]
+      pass = lewt_settings["gmail_password"]
+      app = lewt_settings["google_app_name"]
+      @googleCalender = Google::Calendar.new(
+                                             :username => uname,
+                                             :password => pass,
+                                             :app_name => app
+                                             )
+      super( dStart, dEnd, targetCustomers )
+    end
+    
+    # This method does the actual google calender extract, comparing events to the requested paramters.
+    # It manipulates the @data property of this object which is used by LEWT to gather the extracted data.
+    def extractCalendarData
+      gConf = { :max_results => 2500, :order_by => 'starttime', :single_events => true }
+      @googleCalender.find_events_in_range(@dateStart, @dateEnd + 1, gConf).each do |e|
+        eStart = Time.parse( e.start_time )
+        eEnd = Time.parse( e.end_time )
+        timeDiff = (eEnd - eStart)/60/60
+        target = self.isTargetCustomer?(e.title)
+        if  self.isTargetDate?( eStart ) == true && target != false
+          row = LEWT::LEWTLedger.new({
+                                       :date_start => eStart, 
+                                       :date_end => eEnd, 
+                                       :category => @category, 
+                                       :entity => target["name"], 
+                                       :description => e.content, 
+                                       :quantity => timeDiff, 
+                                       :unit_cost => target["rate"]
+                                     })
+          @data.push(row)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/extensions/calendar-timekeeping/ical_extractor.rb

@@ -0,0 +1,52 @@
+require 'icalendar'
+
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+module CalendarExtractors
+
+  # This class handles extraction from iCal sources.
+  #
+  # ===Usage:
+  # - add the key <tt>ical_filepath</tt> to you settings file corresponding to the filepath of the ical file you wish to have parsed.
+  #
+  class ICalExtractor < CalExtractor
+
+    # Initialises the object and calls the parent class' super() method.
+    def initialize( dateStart, dateEnd, targets, lewt_settings, suppressTargets )
+      @calendarPath = lewt_settings["ical_filepath"]
+      super( dateStart, dateEnd, targets )
+    end
+    
+    # Open iCalender file, parses it, then check events with the regular CalExtractor methods.
+    # Sets the data property of this object if match data is found.
+    def extractCalendarData
+      calendars = Icalendar.parse( File.open( @calendarPath ) )
+      calendars.each do |calendar|
+        calendar.events.each do |e|
+          target = self.isTargetCustomer?( e.summary )
+          dstart = Time.parse( e.dtstart.to_s )
+          dend = Time.parse( e.dtend.to_s )
+          if  self.isTargetDate?(dstart) == true &&  target != false
+            timeDiff = (dend - dstart) /60/60
+            row = LEWT::LEWTLedger.new({
+                                         :date_start => dstart, 
+                                         :date_end => dend, 
+                                         :category => @category, 
+                                         :entity => target["name"], 
+                                         :description => e.description.to_s,
+                                         :quantity => timeDiff, 
+                                         :unit_cost => target["rate"]
+                                       })
+
+            @data.push( row )
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/extensions/liquid-renderer.rb

@@ -0,0 +1,106 @@
+require "liquid"
+require "pdfkit"
+
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+module LEWT
+
+  # The Liquid Renderer LEWT Extension handles rendering processed data to TEXT, HTML, and PDF formats using the
+  # {liquid templating engine}[http://liquidmarkup.org] at its core. This allows for easy marking up of templates
+  # to be used with arbitrary LEWT extensions and processing them into multiple human readable formats on the fly.
+
+  class LiquidRenderer < LEWT::Extension
+    
+    attr_reader :textTemplate, :htmlTemplate, :pdfTemplate, :stylesheet, :markup
+
+    # Sets up this extension and registers its run-time options.
+    def initialize ()
+      options = {
+        :method => {
+          :definition => "Specify html, text, pdf, or any combination of the three to define output method",
+          :default => "text",
+          :short_flag => "-m",
+          :type => String
+        },
+        :save_path => {
+          :definition => "Specify where to save the output file (required for PDFs)",
+          :type => String
+        },
+        :liquid_template => {
+          :definition => "Override the template that liquid render should use. Defaults to the template which matches the processor name but you will want to override this if you are using multiple processors.",
+          :type => String
+        }
+      }
+
+      super({:cmd => "liquid_render", :options => options })
+    end
+
+    
+    # Loads the plaint-text, html, & (optionally) pdf template files of the given template name and parses it with the Liquid class
+    # template [String]:: The name of the template to load.
+    def load_templates ( template )
+      @textTemplate = Liquid::Template::parse( File.open( File.expand_path( lewt_stash  + "/templates/#{template}.text.liquid", __FILE__) ).read )
+      @htmlTemplate = Liquid::Template::parse( File.open( File.expand_path( lewt_stash + "/templates/#{template}.html.liquid", __FILE__) ).read )
+      @stylesheet = File.expand_path( lewt_stash + '/templates/style.css', __FILE__)
+    end
+
+    # Called on LEWT render cycle, this method outputs the data as per a pre-formated liquid template.
+    # options [Hash]:: The options hash passed to this function by the Lewt program.
+    # data [Array]:: An array of hash data to format.
+    def render ( options, data )
+      output = Array.new
+      # template name is always the same as processor name
+      template = options[:liquid_template] != nil ? options[:liquid_template] : options[:process]
+      load_templates( template )
+
+      data.each_with_index do |d, i|
+
+        if options[:method].match "text"
+          r = textTemplate.render(d)
+          if options[:save_path]
+            save_name = format_save_name( options, i )
+            File.open( save_name, 'w') {|f| f.write r }
+            output << save_name
+          else
+            output << r
+          end
+        end
+        
+        if options[:method].match "html"
+          r = htmlTemplate.render(d)
+          if options[:save_path]
+            save_name = format_save_name( options, i )
+            File.open( save_name, 'w') {|f| f.write r }
+            output << save_name
+          else
+            output << r
+          end
+        end
+        
+        if options[:method].match "pdf"
+          raise ArgumentError,"--save-file flag must be specified for PDF output in #{self.class.name}" if !options[:save_path]
+          save_name = format_save_name( options, i )
+          html = htmlTemplate.render(d)
+          kit = PDFKit.new(html, :page_size => 'A4')
+          kit.stylesheets << @stylesheet
+          file = kit.to_file( save_name )
+          output << save_name
+        end
+      end
+      # if options[:dump_output] != false
+      #   output.each do |r|
+      #     puts r
+      #   end
+      # end
+
+      return output
+    end
+    
+    protected
+    
+  end
+
+end

data/lib/extensions/metastat/metamath.rb

@@ -0,0 +1,108 @@
+
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+module LEWT
+
+  # MetaMath is a base class for developing statistics routines for use with the Metastat LEWT extension
+  # It provides some convienience methods for developing new routines, as well as acting as a container
+  # for common mathematical proceedures.
+
+  class MetaMath
+    
+    # Takes an array of numeric values and returns there mean.
+    # ar:: Array of values
+    def mean(ar)
+      raise TypeError "Expected an array" if not ar.kind_of?(Array)
+      total = ar.reduce(0) { |sum, x| x + sum }
+      return Float(total)/Float(ar.length)
+    end
+    
+    # Takes an array of values and returns there mode.
+    # ar:: Array of values
+    def mode(ar)
+      raise TypeError "Expected an array" if not ar.kind_of?(Array)
+      freq = ar.inject(Hash.new(0)) { |h,v| h[v] += 1; h }
+      return ar.max_by { |v| freq[v] }
+    end
+    
+    # Takes an array of values and returns the median
+    # ar:: Array of values
+    def median(ar)
+      raise TypeError "Expected an array" if not ar.kind_of?(Array)
+      mid = ar.length / 2
+      return (mid % 1 != 0) ? mean( [ ar[(mid).floor], ar[(mid).round ]] ) : ar[mid]
+    end
+
+    # Return the descriptive statistics for an array of values [mean, media, mode]
+    # ar:: Array of values
+    def descriptive_stats(ar)
+      raise TypeError "Expected an array" if not ar.kind_of?(Array)
+      return {
+        :mean => mean(ar),
+        :median => median(ar),
+        :mode => mode(ar)
+      }
+    end
+
+  end
+
+  # This subclass performs a Pearson Correlation analysis on an x/y dataset.
+  class PearsonR < MetaMath
+    
+    def initialize (xs, ys)
+      raise Exception "_x & _y datasets must be of equal length" if xs.length != ys.length
+      @xs, @ys = xs, ys
+    end
+
+    # Returns a Pearson Correlation R value
+    # xs:: The x series array of values
+    # ys:: The y series array of values
+    def correlate(xs = @xs, ys = @ys)
+      raise Exception "_x & _y datasets must be of equal length" if xs.length != ys.length
+      x_mean = mean(@xs)
+      y_mean = mean(@ys)
+      numerator = (0...@xs.length).reduce(0) do |sum, i|
+        sum + ((@xs[i] - x_mean) * (@ys[i] - y_mean))
+      end
+      denominator = @xs.reduce(0) do |sum, x|
+        sum + ((x - x_mean) ** 2)
+      end      
+      (numerator / Math.sqrt(denominator))
+    end
+    
+  end
+
+
+  # This class performas a simple regression on an x/y dataset
+  class SimpleRegression < MetaMath
+
+    def initialize (xs, ys)
+      raise Exception "_x & _y datasets must be of equal length" if xs.length != ys.length
+      @xs, @ys = xs, ys
+    end
+
+    def y_intercept
+      mean(@ys) - (slope * mean(@xs))
+    end
+    
+    def slope
+      x_mean = mean(@xs)
+      y_mean = mean(@ys)
+      
+      numerator = (0...@xs.length).reduce(0) do |sum, i|
+        sum + ((@xs[i] - x_mean) * (@ys[i] - y_mean))
+      end
+      
+      denominator = @xs.reduce(0) do |sum, x|
+        sum + ((x - x_mean) ** 2)
+      end
+      
+      (numerator / denominator)
+    end
+
+
+  end
+end

data/lib/extensions/metastat/metastat.rb

@@ -0,0 +1,161 @@
+require_relative "metamath.rb"
+
+# Author::    Jason Wijegooneratne  (mailto:code@jwije.com)
+# Copyright:: Copyright (c) 2014 Jason Wijegooneratne
+# License::   MIT. See LICENSE.md distributed with the source code for more information.
+
+
+module LEWT
+
+  # The Metastat extension [experimental] handles processing meta-data added to ledger entries. It can compute statistics based of
+  # these parsed values
+  #
+  # ===Usage:
+  # Here is an example 'description' field entered into a LEWTLedger which can be parsed with Metastat
+  #
+  #  'description' => 'Adding some features to the system #happiness=10 #average-pay'
+  #
+  # Metastat will extract the # tags from the above string and log an indicatior called happinesswhich is equal to 10
+  # and a Boolean switch average-pay will be set to true
+  #
+  # These values will be aggregate over the whole 'extract' dataset then processed in a few ways.
+  #
+  # 1. Create a graph out of the data
+  # 2. Create a summary table out of the data 
+
+  class Metastat < LEWT::Extension
+
+    attr_reader :client_table, :client_graph
+
+    # Sets up this extension and registers its options
+    def initialize
+      options = {
+        :tags => {
+          :definition => "A comma seperated list of metatags to lookup.",
+          :type => String
+        },
+        :y_series => {
+          :definition => "An optional y series to use for a correlation analysis.",
+          :type => String
+        }
+      }
+      @boolean_table = Hash.new
+      @raw_data = Hash.new
+      @dataset = Hash.new
+      super({:cmd => 'metastat', :options => options})
+    end
+    
+    # Handles the process event for this extension
+    # options [Hash]:: The options hash passed to this function by the Lewt program.
+    # extract_data [LEWTBook]:: The data in LEWTBook format
+    def process (options, extract_data)
+      @options = options
+      extract_data.each do |row|
+        filter_row_values(row, options)
+      end
+      @dataset["frequency_table"] = @boolean_table
+      if options[:y_series]
+        @dataset["correlations"] = compute_correlations @raw_data
+      end
+      return @dataset
+    end
+
+    protected
+    
+    # extracts row data from a LEWTLedger object. Also handles initiating tag lookup.
+    # row [LEWTLedger]:: a LEWTLedger object container the row data.
+    # options [Hash]:: a Hash containing the options passed to LEWT.
+    def filter_row_values (row, options)
+      return if row.metatags == nil
+      # match tags requested via options
+      options[:tags].split(Lewt::OPTION_DELIMITER_REGEX).each do |meta|
+        row.metatags.each { |k, v|
+          next unless meta.gsub(Lewt::OPTION_SYMBOL_REGEX,"_").to_sym == k
+          # found match operate on it
+          client = get_matched_customers(row[:entity])[0]
+          # case our value
+          case v
+          when !!v == v
+            add_to_tally( client, row, k, v )
+          when Rational
+            transform_dataset( client, row, k, v)
+          end
+        }
+      end
+    end
+    
+    # adds +1 count to the client table for the provided key
+    # c [Hash]:: The client hash
+    # r [LEWTLedger]:: A LEWTLedger row
+    # k:: the metatag hash key
+    # v:: the meta tag value, this should be a boolean
+    def add_to_tally( c, r, k, v )
+      if !@boolean_table.has_key?(c["name"])
+        @boolean_table[ c["name"] ] = { k.to_s => 0 }
+      end
+
+      if !@boolean_table[c["name"]].has_key?(k.to_s)
+        @boolean_table[c["name"]][k.to_s] = 0;
+      end
+      
+      @boolean_table[c["name"]][k.to_s] += 1
+    end
+
+    # Transforms the dataset into something usable for statistical computation
+    # c [Hash]:: The client hash
+    # r [LEWTLedger]:: A LEWTLedger row
+    # k:: the metatag hash key
+    # v:: the meta tag value, this should be a boolean
+    def transform_dataset( c, r, k, v )
+      if !@raw_data.has_key?(c["name"])
+        @raw_data[ c["name"] ] = Array.new
+      end
+      # prep the data
+      @raw_data[c["name"]].push(prepare_row_data(r,k,v))
+    end
+    
+    # prepares normal row for statistical analysis computing some basic numbers we are going to need
+    # r [LEWTLedger]:: A LEWTLedger row
+    # v:: the meta tag value, this should be a boolean
+    def prepare_row_data ( r, k, v )
+      data = {
+        k.to_sym => v.to_f,
+        :duration => r[:quantity].to_f,
+        :value => r[:total].to_f
+      }
+      return data
+    end
+    
+    # performs some statistics using R. 
+    # d:: the dataset to work with
+    def compute_correlations(d)
+      result = nil
+      d.each { |context, data_array|
+        # hash of arrays
+        data_hash = Hash.new
+        data_array.each do |r|
+          r.each { |k,v|
+            data_hash[k] = Array.new if !data_hash.has_key? k
+            data_hash[k].push v
+          }
+        end
+        result = correlate_y data_hash, @options[:y_series]
+      }
+      return result
+    end
+    
+    def correlate_y(r_dataset, y_key)
+      results = Hash.new
+      r_dataset.each { |k, v_set|
+        next if k == y_key.to_sym
+        results[y_key.to_sym] = Hash.new if results[y_key.to_sym] == nil
+        results[y_key.to_sym][k] = Hash.new if results[y_key.to_sym][k] == nil
+        r = PearsonR.new( v_set, r_dataset[y_key.to_sym] )
+        results[y_key.to_sym][k][:pearson_r] = r.correlate
+        results[y_key.to_sym][k][:descriptive_stats] = r.descriptive_stats(v_set)
+      }
+      return results
+    end
+
+  end
+end