matt 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 712f1cd0be5efa12e62fe0491c0c14028b76a5e4edcdaf792bf8ab9652397218
+  data.tar.gz: e133df58cfba2501cdd806668ff6e21363150a32a9563a6fa554938dc2bfed5c
+SHA512:
+  metadata.gz: ae322c7f52f9bab450bb21781b5653bd87a6e40703fadfe2768cc56305508464945f411c15584b3702cfefc969baf2af0fe4e437b6b454355b934b7602d39920
+  data.tar.gz: d4ef94dc91c994bb4886a4b9c4beb0c863af6d62bc2c66c440fb0a594788640bca2c536bbe1f86a8f3e22ebb93bc357ef09e0e29c604256aa4f064b167ba33ef

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/README.md

@@ -0,0 +1,115 @@
+# Matt - A monitoring tool for data-driven business decisions
+
+Matt is a CTO-oriented tool that helps capturing daily data points that are
+relevant for conducting growth hacking and other data-driven decisions.
+
+It consists of the following abstractions:
+
+* Datasource: provides access to data from which measures are extracted
+* Measure: standardized data vector for relevant decisions
+* Exporter: a way to save measures for future analysis and visualization
+
+## How to install Matt?
+
+There are various usage scenario to use matt, according to your needs and
+types of environment and datasources.
+
+### As a commandline
+
+To simply use Matt as a commandline and connect to local databases, simply
+install and use the ruby gem:
+
+```
+gem install matt
+matt --help
+```
+
+### As a docker image
+
+If you don't have ruby installed and want to use docker instead, we provide a
+docker image that has Matt as entrypoint.
+
+```
+docker run enspirit/matt --help
+```
+
+Please check docker's network documentation if you want Matt to connect to
+databases running on your host.
+
+### In a Gemfile
+
+It may be useful to use Matt as part of a larger ruby gemset, for instance
+because you need various ruby gems to connect to mysql, postgres, or whatever.
+
+Then in your Gemfile:
+
+```
+source https://rubygems.org
+
+gem "matt"
+```
+
+Then,
+
+```
+bundle exec matt --help
+```
+
+### As a docker base image + a Gemfile
+
+If Matt is part of a larger docker/kubernetes infrastructure, you probably need
+the last scenario while also having your own docker image for the component.
+
+This is a good basis for a Dockerfile:
+
+```
+FROM enspirit/matt   # or from ruby-3.0:alpine
+
+COPY Gemfile .
+RUN bundle install
+
+ENTRYPOINT ['bundle', 'exec', 'matt']
+```
+
+Let's say you build that image as `my/matt`, you can now simply run matt as
+follows:
+
+```
+docker run my/matt --help
+```
+
+In such a scenario, you may want to use Matt's version that exists in the
+docker container instead without having to specify it in the Gemfile.
+The following Gemfile hack works:
+
+```
+gem "matt", path: "/matt"
+```
+
+## Semantics versioning
+
+Matt uses semantics versioning since 1.0.
+
+The public API consists in:
+
+* Matt public methods
+* Matt::Configuration public methods
+* Matt::Datasource public & protected methods exposed to including classes
+* Matt::Exporter (same)
+* Matt::Measure (same)
+* `matt` commandline and its options
+* `enspirit/matt` docker image with `/matt` and `/home/matt` behavior
+
+## Contribute
+
+Please use github issues and pull requests for all questions, bug reports,
+and contributions. Don't hesitate to get in touch with us with an early code
+spike if you plan to add non trivial features.
+
+## Licence
+
+This software is distributed by Enspirit SRL under a MIT Licence. Please
+contact Bernard Lambeau (blambeau@gmail.com) with any question.
+
+Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
+actively using and contributing to the library.

data/bin/matt

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'matt'
+exitcode = Matt::Command.new.call(ARGV)
+exit(exitcode)

data/lib/matt.rb

@@ -0,0 +1,39 @@
+require 'date'
+require 'ostruct'
+require 'json'
+require 'sequel'
+require 'bmg'
+require 'path'
+require 'bmg/sequel'
+module Matt
+
+  def env(which, default = nil)
+    val = ENV.has_key?(which) ? ENV[which] : default
+    val = val.strip if val.is_a?(String)
+    val
+  end
+  module_function :env
+
+  def alltime_predicate
+    Predicate.tautology
+  end
+  module_function :alltime_predicate
+
+  def yesterday_predicate
+    Predicate.gte(:at, Date.today - 1) & Predicate.lt(:at, Date.today)
+  end
+  module_function :yesterday_predicate
+
+  def today_predicate
+    Predicate.gte(:at, Date.today) & Predicate.lt(:at, Date.today + 1)
+  end
+  module_function :today_predicate
+
+end # module Matt
+require_relative 'matt/version'
+require_relative 'matt/support'
+require_relative 'matt/datasource'
+require_relative 'matt/measure'
+require_relative 'matt/exporter'
+require_relative 'matt/configuration'
+require_relative 'matt/command'

data/lib/matt/command.rb

@@ -0,0 +1,211 @@
+#/
+#/ Matt - A monitoring tool for data-driven business decisions
+#/ vVERSION - (c) Enspirit SRL, 2021 and beyond
+#/
+#/ Usage: matt [-f folder] command [options...] [args...]
+#/
+#/ show measure           Show the datapoints of a given measure
+#/ export measure*        Export all (or specific) measure(s) to
+#/ ping datasource*       Ping all (or specific) datasource(s)
+#/                        their default exporters
+#/
+#/ -f FOLDER              Use the folder as entry point (`pwd` by default)
+#/     --all-time         Do not restrict measures shown/exported
+#/     --today            Only show/export measures for today
+#/     --yesterday        Only show/export measures for yesterday's (default)
+#/     --to=exporter,...  Override the default exporters
+#/     --json             Use json when displaying measures on console
+#/     --csv              Use csv when displaying measures on console (default)
+#/ -h, --help             Show this help message
+#/     --version          Show matt version
+#/
+#/ Examples:
+#/     matt show --all-time --json account_creations
+#/     matt export --today --to=sql,prometheus account_creations
+#/
+require 'optparse'
+module Matt
+  class Command
+    include Support::Puts
+
+    attr_accessor :exitcode
+    attr_accessor :output_format
+    attr_accessor :to
+
+    def initialize
+      @output_format = :csv
+      @to = nil
+      yield(self) if block_given?
+    end
+
+    def on_configuration(&bl)
+      @on_configuration = bl
+      self
+    end
+
+    def configuration
+      return @configuration if @configuration
+      self.configuration = Configuration.new(Path.pwd)
+    end
+
+    def configuration=(c)
+      @on_configuration.call(c) if @on_configuration
+      @configuration = c
+    end
+
+    def has_configuration?
+      !!@configuration
+    end
+
+    def parse_argv(argv)
+      opt_parser.parse!(argv)
+    end
+
+    def call(argv)
+      ok = false
+      catch :abort do
+        parse_argv(argv)
+        _call(argv)
+        ok = true
+      end
+      self.exitcode = ok ? 0 : 1
+    end
+
+  protected
+
+    def _call(argv)
+      if argv.empty?
+        puts_out opt_parser
+        abort
+      end
+
+      meth = :"do_#{argv.first}"
+      if self.respond_to?(meth, true)
+        send(meth, argv[1..-1])
+      else
+        puts_err "No such command #{argv.first}"
+        abort
+      end
+    end
+
+    def do_show(argv)
+      argv_count!(argv, 1)
+      m = measure_exists!(argv.first)
+      data = m.full_data.restrict(configuration.at_predicate)
+      case of = output_format
+      when :json
+        puts_out JSON.pretty_generate(data)
+      when :csv
+        puts_out data.to_csv(configuration.csv_options)
+      else
+        puts_err "Unknown format #{of}"
+      end
+    end
+
+    def do_ping(argv)
+      which_ones = if argv.empty?
+        configuration.datasources.to_h.values
+      else
+        argv.map{|arg| datasource_exists!(arg) }
+      end
+      which_ones.each do |d|
+        d.ping
+      end
+    end
+
+    def do_export(argv)
+      argv_count!(argv, 1)
+      m = measure_exists!(argv.first)
+      data = m.full_data.restrict(configuration.at_predicate)
+      (@to || m.exporters).each do |e|
+        exporter = exporter_exists!(e)
+        exporter.export(m, data)
+      end
+    end
+
+    def do_help(*args)
+      file = __FILE__
+      exec "grep ^#/<'#{file}'|cut -c4-|sed s/VERSION/#{Matt::VERSION}/g"
+    end
+
+  protected
+
+    def opt_parser
+      OptionParser.new do |opts|
+        opts.banner = "Usage: matt [options] COMMAND [args]"
+        opts.on("-f FOLDER") do |folder|
+          p = Path(folder)
+          if has_configuration?
+            puts_err "-f must be used before other configuration options"
+            abort
+          elsif p.exists? && p.directory?
+            self.configuration = Configuration.new(p)
+          else
+            puts_err "No such folder: #{folder}"
+            abort
+          end
+        end
+        opts.on("--all-time") do
+          self.configuration.at_predicate = Matt.alltime_predicate
+        end
+        opts.on("--yesterday") do
+          self.configuration.at_predicate = Matt.yesterday_predicate
+        end
+        opts.on("--today") do
+          self.configuration.at_predicate = Matt.today_predicate
+        end
+        opts.on("--to=EXPORTERS") do |exporters|
+          @to = (@to || []) + exporters.split(/\s*,\s*/).map{|e|
+            exporter_exists!(e.to_sym).name
+          }
+        end
+        opts.on("--json") do
+          self.output_format = :json
+        end
+        opts.on("--csv") do
+          self.output_format = :csv
+        end
+        opts.on('--version', "Show version number") do
+          puts_out "Matt v#{VERSION} - (c) Enspirit SRL"
+          abort
+        end
+        opts.on("-h", "--help", "Prints this help") do
+          do_help
+          abort
+        end
+      end
+    end
+
+    def argv_count!(argv, n)
+      return if argv.size == n
+      puts_err "#{n} arguments expected, got #{argv.size}"
+      abort
+    end
+
+    def measure_exists!(name)
+      m = configuration.measures.send(name.to_sym)
+      return m if m
+      puts_err "No such measure #{name}"
+      abort
+    end
+
+    def exporter_exists!(name)
+      m = configuration.exporters.send(name.to_sym)
+      return m if m
+      puts_err "No such exporter #{name}"
+      abort
+    end
+
+    def datasource_exists!(name)
+      d = configuration.datasources.send(name.to_sym)
+      return d if d
+      puts_err "No such datasource #{name}"
+      abort
+    end
+
+    def abort
+      throw :abort
+    end
+
+  end # class Command
+end # module Matt

data/lib/matt/configuration.rb

@@ -0,0 +1,69 @@
+module Matt
+  class Configuration
+
+    def initialize(folder)
+      @folder = folder
+      @stdout = $stdout
+      @stderr = $stderr
+      @csv_options = {
+        :write_headers => true
+      }
+      @at_predicate = Matt.yesterday_predicate
+      yield(self) if block_given?
+    end
+    attr_accessor :folder
+    attr_accessor :stdout, :stderr
+    attr_accessor :csv_options
+    attr_accessor :at_predicate
+
+    def datasources_folder
+      folder/"datasources"
+    end
+
+    def datasources
+      @datasources ||= load_dynamic_objects(datasources_folder, Matt::Datasource)
+    end
+    alias :ds :datasources
+
+    def measures_folder
+      folder/"measures"
+    end
+
+    def measures
+      @measures ||= load_dynamic_objects(measures_folder, Matt::Measure)
+    end
+    alias :ms :measures
+
+    def exporters_folder
+      folder/"exporters"
+    end
+
+    def exporters
+      @exporters ||= load_dynamic_objects(exporters_folder, Matt::Exporter)
+    end
+    alias :es :exporters
+
+  private
+
+    def load_dynamic_objects(folder, expected_type)
+      struct = OpenStruct.new
+      folder.glob("*.rb").each_with_object(struct) do |file,memo|
+        obj = load_dynamic_object(file)
+        if obj.nil?
+          raise "Wrong file #{file}: eval returned nil"
+        elsif !obj.is_a?(expected_type)
+          raise "Wrong file #{file}: must include #{expected_type}"
+        else
+          obj.configuration = self
+          obj.name = file.basename.rm_ext.to_s
+          memo.send("#{obj.name}=", obj)
+        end
+      end
+    end
+
+    def load_dynamic_object(file)
+      self.instance_eval(file.read, file.to_s)
+    end
+
+  end # class Configuration
+end # module Matt

data/lib/matt/datasource.rb

@@ -0,0 +1,14 @@
+module Matt
+  module Datasource
+    include Support::Puts
+
+    attr_accessor :name
+    attr_accessor :configuration
+
+    def ping
+      raise NotImplementedError, "#{self} should implement `ping`"
+    end
+
+  end # module Datasource
+end # module Matt
+require_relative 'datasource/sql'

data/lib/matt/datasource/sql.rb

@@ -0,0 +1,33 @@
+module Matt
+  module Datasource
+    class Sql
+      include Matt::Datasource
+
+      def initialize(config)
+        @config = config
+        @sequel_db = ::Sequel.connect(config)
+      end
+      attr_reader :config, :sequel_db
+
+      def ping
+        sequel_db.test_connection
+        puts "#{self} -- Ok."
+      rescue => ex
+        puts_err "#{self} -- Ko: #{ex.message}"
+      end
+
+      def to_s
+        if config.is_a?(Hash)
+          "#{config[:host]}:#{config[:port]}/#{config[:database]}"
+        else
+          config.to_s
+        end
+      end
+
+      def sequel(*args)
+        Bmg.sequel(*args)
+      end
+
+    end
+  end
+end

data/lib/matt/exporter.rb

@@ -0,0 +1,10 @@
+module Matt
+  module Exporter
+    include Support::Puts
+
+    attr_accessor :name
+    attr_accessor :configuration
+
+  end # module Exporter
+end # module Matt
+require_relative 'exporter/sql'

data/lib/matt/exporter/sql.rb

@@ -0,0 +1,90 @@
+module Matt
+  module Exporter
+    class Sql
+      include Matt::Exporter
+
+      def initialize(arg)
+        case @config = arg
+        when String
+        when Hash
+        when Sequel::Database
+        when Matt::Datasource::Sql
+        else
+          raise ArgumentError, "Unable to use `#{arg}` to create an Sql exporter"
+        end
+      end
+      attr_reader :config
+
+      def sequel_db
+        @sequel_db ||= case config
+        when Sequel::Database      then config
+        when Matt::Datasource::Sql then config.sequel_db
+        else Sequel.connect(config)
+        end
+      end
+
+      def export(measure, data)
+        now = Time.now
+        data = data.constants(:created_at => now)
+        table = ensure_table!(measure)
+        remove_old_data!(table, data)
+        insert_new_data!(table, data)
+      end
+
+    private
+
+      def remove_old_data!(table, data)
+        dates = data.project([:at]).map{|t| t[:at] }
+        table.where(:at => dates).delete
+      end
+
+      def insert_new_data!(table, data)
+        table.multi_insert(data.to_a)
+      end
+
+      def ensure_table!(measure)
+        tb_name = measure.name.to_sym
+        fields = [:at] + measure.dimensions.keys + measure.metrics.keys
+        if sequel_db.table_exists?(tb_name)
+          align_table!(tb_name, measure, fields)
+        else
+          create_table!(tb_name, measure, fields)
+        end
+        sequel_db[tb_name]
+      end
+
+      def align_table!(tb_name, measure, fields)
+        schema   = sequel_db.schema(tb_name)
+        existing = schema.map{|s| s.first } - [:created_at]
+        missing  = fields - existing
+        missing  = measure.dimensions.merge(measure.metrics).select{|x|
+          missing.include?(x)
+        }
+        outdated = existing - fields
+        return if missing.empty? && outdated.empty?
+        sequel_db.alter_table(tb_name) do
+          outdated.each do |out|
+            drop_column(out)
+          end
+          missing.each_pair do |mis, type|
+            add_column(mis, type, null: true)
+          end
+        end
+      end
+
+      def create_table!(tb_name, measure, fields)
+        sequel_db.create_table(tb_name) do
+          column(:at, Date, :null => false)
+          column(:created_at, :timestamp, :null => false)
+          measure.dimensions.each do |name,type|
+            column(name, type, :null => true)
+          end
+          measure.metrics.each do |name,type|
+            column(name, type, :null => true)
+          end
+        end
+      end
+
+    end # class Sql
+  end # module Exporter
+end # module Matt

data/lib/matt/measure.rb

@@ -0,0 +1,25 @@
+module Matt
+  module Measure
+    include Support::Puts
+
+    attr_accessor :name
+    attr_accessor :configuration
+
+    def dimensions
+      {}
+    end
+
+    def exporters
+      []
+    end
+
+    def metrics
+      raise NotImplementedError, "#{self} must implement `metrics`"
+    end
+
+    def ds
+      configuration.datasources
+    end
+
+  end # module Measure
+end # module Matt

data/lib/matt/support.rb

@@ -0,0 +1 @@
+require_relative 'support/puts'

data/lib/matt/support/puts.rb

@@ -0,0 +1,26 @@
+module Matt
+  module Support
+    module Puts
+
+      def stdout
+        configuration.stdout
+      end
+
+      def stderr
+        configuration.stderr
+      end
+
+      def puts_out(*args, &bl)
+        return $stdout.puts(*args, &bl) unless configuration
+        configuration.stdout.puts(*args, &bl)
+      end
+      alias :puts :puts_out
+
+      def puts_err(*args, &bl)
+        return $stderr.puts(*args, &bl) unless configuration
+        configuration.stderr.puts(*args, &bl)
+      end
+
+    end # module Puts
+  end # module Support
+end # module Matt

data/lib/matt/version.rb

@@ -0,0 +1,8 @@
+module Matt
+  module Version
+    MAJOR = 1
+    MINOR = 0
+    TINY  = 0
+  end # module Version
+  VERSION = [Version::MAJOR, Version::MINOR, Version::TINY].join(".")
+end # module Matt

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification
+name: matt
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Bernard Lambeau
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-06-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: path
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bmg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.19'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '14.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '14.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Matt help monitoring data points for better business decision.
+email: blambeau@gmail.com
+executables:
+- matt
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- bin/matt
+- lib/matt.rb
+- lib/matt/command.rb
+- lib/matt/configuration.rb
+- lib/matt/datasource.rb
+- lib/matt/datasource/sql.rb
+- lib/matt/exporter.rb
+- lib/matt/exporter/sql.rb
+- lib/matt/measure.rb
+- lib/matt/support.rb
+- lib/matt/support/puts.rb
+- lib/matt/version.rb
+homepage: http://github.com/enspirit/matt
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key: 
+specification_version: 4
+summary: Matt help monitoring data points for better business decision.
+test_files: []