timescaledb 0.1.0 → 0.1.5

data/lib/timescale/acts_as_hypertable/core.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Timescale
+  module ActsAsHypertable
+    module Core
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def time_column
+          @time_column ||= hypertable_options[:time_column] || :created_at
+        end
+
+        protected
+
+        def define_association_scopes
+          scope :chunks, -> do
+            Chunk.where(hypertable_name: table_name)
+          end
+
+          scope :hypertable, -> do
+            Hypertable.find_by(hypertable_name: table_name)
+          end
+
+          scope :jobs, -> do
+            Job.where(hypertable_name: table_name)
+          end
+
+          scope :job_stats, -> do
+            JobStats.where(hypertable_name: table_name)
+          end
+
+          scope :compression_settings, -> do
+            CompressionSettings.where(hypertable_name: table_name)
+          end
+
+          scope :continuous_aggregates, -> do
+            ContinuousAggregates.where(hypertable_name: table_name)
+          end
+        end
+
+        def define_default_scopes
+          scope :previous_month, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.last_month.in_time_zone.beginning_of_month.to_date,
+              end_time: Date.today.last_month.in_time_zone.end_of_month.to_date
+            )
+          end
+
+          scope :previous_week, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.last_week.in_time_zone.beginning_of_week.to_date,
+              end_time: Date.today.last_week.in_time_zone.end_of_week.to_date
+            )
+          end
+
+          scope :this_month, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.in_time_zone.beginning_of_month.to_date,
+              end_time: Date.today.in_time_zone.end_of_month.to_date
+            )
+          end
+
+          scope :this_week, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.in_time_zone.beginning_of_week.to_date,
+              end_time: Date.today.in_time_zone.end_of_week.to_date
+            )
+          end
+
+          scope :yesterday, -> { where("DATE(#{time_column}) = ?", Date.yesterday.in_time_zone.to_date) }
+          scope :today, -> { where("DATE(#{time_column}) = ?", Date.today.in_time_zone.to_date) }
+          scope :last_hour, -> { where("#{time_column} > ?", 1.hour.ago.in_time_zone) }
+        end
+
+        def normalize_hypertable_options
+          hypertable_options[:time_column] = hypertable_options[:time_column].to_sym
+        end
+      end
+    end
+  end
+end

data/lib/timescale/acts_as_hypertable.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Timescale
+  # If you want your model to hook into its underlying hypertable
+  # as well as have access to TimescaleDB specific data, methods, and more,
+  # specify this macro in your model.
+  #
+  # @note Your model's table needs to have already been converted to a hypertable
+  # via the TimescaleDB `create_hypertable` function for this to work.
+  #
+  # @see https://docs.timescale.com/api/latest/hypertable/create_hypertable/ for
+  #   how to use the SQL `create_hypertable` function.
+  # @see Timescale::MigrationHelpers#create_table for how to create a new hypertable
+  # via a Rails migration utilizing the standard `create_table` method.
+  #
+  # @example Enabling the macro on your model
+  #   class Event < ActiveRecord::Base
+  #     acts_as_hypertable
+  #   end
+  #
+  # @see Timescale::ActsAsHypertable#acts_as_hypertable
+  #   for configuration options
+  module ActsAsHypertable
+    DEFAULT_OPTIONS = {
+      time_column: :created_at
+    }.freeze
+
+    def acts_as_hypertable?
+      included_modules.include?(Timescale::ActsAsHypertable::Core)
+    end
+
+    # == Configuration options
+    #
+    # @param [Hash] options The options to initialize your macro with.
+    # @option options [Symbol] :time_column The name of the column in your
+    #   model's table containing time values. The name provided should be
+    #   the same name as the `time_column_name` you passed to the
+    #   TimescaleDB `create_hypertable` function.
+    #
+    # @example Enabling the macro on your model with options
+    #   class Event < ActiveRecord::Base
+    #     acts_as_hypertable time_column: :timestamp
+    #   end
+    #
+    def acts_as_hypertable(options = {})
+      return if acts_as_hypertable?
+
+      include Timescale::ActsAsHypertable::Core
+
+      class_attribute :hypertable_options, instance_writer: false
+
+      self.hypertable_options = DEFAULT_OPTIONS.dup
+      hypertable_options.merge!(options)
+      normalize_hypertable_options
+
+      define_association_scopes
+      define_default_scopes
+    end
+  end
+end
+
+ActiveRecord::Base.extend Timescale::ActsAsHypertable

data/lib/timescale/chunk.rb

@@ -8,6 +8,14 @@ module Timescale
     scope :compressed, -> { where(is_compressed: true) }
     scope :uncompressed, -> { where(is_compressed: false) }
 
+    scope :resume, -> do
+      {
+        total: count,
+        compressed: compressed.count,
+        uncompressed: uncompressed.count
+      }
+    end
+
     def compress!
       execute("SELECT compress_chunk(#{chunk_relation})")
     end

data/lib/timescale/compression_settings.rb

@@ -1,6 +1,7 @@
 module Timescale
-  class CompressionSettings < ActiveRecord::Base
+  class CompressionSetting < ActiveRecord::Base
     self.table_name = "timescaledb_information.compression_settings"
     belongs_to :hypertable, foreign_key: :hypertable_name
   end
+  CompressionSettings = CompressionSetting
 end

data/lib/timescale/continuous_aggregates.rb

@@ -1,9 +1,19 @@
 module Timescale
-  class ContinuousAggregates < ActiveRecord::Base
+  class ContinuousAggregate < ActiveRecord::Base
     self.table_name = "timescaledb_information.continuous_aggregates"
     self.primary_key = 'materialization_hypertable_name'
 
     has_many :jobs, foreign_key: "hypertable_name",
       class_name: "Timescale::Job"
+
+    has_many :chunks, foreign_key: "hypertable_name",
+      class_name: "Timescale::Chunk"
+
+    scope :resume, -> do
+      {
+        total: count
+      }
+    end
   end
+  ContinuousAggregates = ContinuousAggregate
 end

data/lib/timescale/dimensions.rb

@@ -0,0 +1,6 @@
+module Timescale
+  class Dimension < ActiveRecord::Base
+    self.table_name = "timescaledb_information.dimensions"
+  end
+  Dimensions = Dimension
+end

data/lib/timescale/hypertable.rb

@@ -1,7 +1,6 @@
 module Timescale
   class Hypertable < ActiveRecord::Base
     self.table_name = "timescaledb_information.hypertables"
-
     self.primary_key = "hypertable_name"
 
     has_many :jobs, foreign_key: "hypertable_name"
@@ -9,18 +8,39 @@ module Timescale
 
     has_many :compression_settings,
       foreign_key: "hypertable_name",
-      class_name: "Timescale::CompressionSettings"
+      class_name: "Timescale::CompressionSetting"
+
+    has_one :dimensions,
+      foreign_key: "hypertable_name",
+      class_name: "Timescale::Dimension"
 
     has_many :continuous_aggregates,
       foreign_key: "hypertable_name",
-      class_name: "Timescale::ContinuousAggregates"
+      class_name: "Timescale::ContinuousAggregate"
 
-    def detailed_size
+    def chunks_detailed_size
       struct_from "SELECT * from chunks_detailed_size('#{self.hypertable_name}')"
     end
 
+    def approximate_row_count
+      struct_from("SELECT * FROM approximate_row_count('#{self.hypertable_name}')").first.approximate_row_count
+    end
+
     def compression_stats
-      struct_from "SELECT * from hypertable_compression_stats('#{self.hypertable_name}')"
+      @compression_stats ||=
+        struct_from("SELECT * from hypertable_compression_stats('#{self.hypertable_name}')").first || {}
+    end
+
+    def detailed_size
+      struct_from("SELECT * FROM hypertable_detailed_size('#{self.hypertable_name}')").first
+    end
+
+    def before_total_bytes
+      compression_stats["before_compression_total_bytes"] || detailed_size.total_bytes
+    end
+
+    def after_total_bytes
+      compression_stats["after_compression_total_bytes"] || 0
     end
 
     private

data/lib/timescale/job.rb

@@ -3,11 +3,8 @@ module Timescale
     self.table_name = "timescaledb_information.jobs"
     self.primary_key = "job_id"
 
-    attribute :schedule_interval, :interval
-    attribute :max_runtime, :interval
-    attribute :retry_period, :interval
-
-    scope :compression, -> { where(proc_name: "tsbs_compress_chunks") }
+    scope :compression, -> { where(proc_name: [:tsbs_compress_chunks, :policy_compression]) }
+    scope :refresh_continuous_aggregate, -> { where(proc_name: :policy_refresh_continuous_aggregate) }
     scope :scheduled, -> { where(scheduled: true) }
   end
 end

data/lib/timescale/job_stats.rb

@@ -1,12 +1,17 @@
 module Timescale
-  class JobStats < ActiveRecord::Base
+  class JobStat < ActiveRecord::Base
     self.table_name = "timescaledb_information.job_stats"
 
     belongs_to :job
 
-    attribute :last_run_duration, :interval
-
     scope :success, -> { where(last_run_status: "Success") }
     scope :scheduled, -> { where(job_status: "Scheduled") }
+    scope :resume, -> do
+      select("sum(total_successes)::int as success,
+             sum(total_runs)::int as runs,
+             sum(total_failures)::int as failures")
+        .to_a.map{|e|e.attributes.transform_keys(&:to_sym) }
+    end
   end
+  JobStats = JobStat
 end

data/lib/timescale/migration_helpers.rb

@@ -21,12 +21,12 @@ module Timescale
     #  end
     def create_table(table_name, id: :primary_key, primary_key: nil, force: nil, **options)
       super
-      setup_hypertable_options(table_name, **options[:hypertable]) if options.key?(:hypertable)
+      create_hypertable(table_name, **options[:hypertable]) if options.key?(:hypertable)
     end
 
     # Setup hypertable from options
     # @see create_table with the hypertable options.
-    def setup_hypertable_options(table_name,
+    def create_hypertable(table_name,
                                  time_column: 'created_at',
                                  chunk_time_interval: '1 week',
                                  compress_segmentby: nil,
@@ -51,7 +51,28 @@ module Timescale
       end
     end
 
-    def create_continuous_aggregates(name, query, **options)
+    # Create a new continuous aggregate
+    #
+    # @param name [String, Symbol] The name of the continuous aggregate.
+    # @param query [String] The SQL query for the aggregate view definition.
+    # @param with_data [Boolean] Set to true to create the aggregate WITH DATA
+    # @param refresh_policies [Hash] Set to create a refresh policy
+    # @option refresh_policies [String] start_offset: INTERVAL or integer
+    # @option refresh_policies [String] end_offset: INTERVAL or integer
+    # @option refresh_policies [String] schedule_interval: INTERVAL
+    #
+    # @see https://docs.timescale.com/api/latest/continuous-aggregates/add_continuous_aggregate_policy/
+    #
+    # @example
+    #   create_continuous_aggregate(:activity_counts, query: <<-SQL, refresh_policies: { schedule_interval: "INTERVAL '1 hour'" })
+    #     SELECT
+    #       time_bucket(INTERVAL '1 day', activity.created_at) AS bucket,
+    #       count(*)
+    #     FROM activity
+    #     GROUP BY bucket
+    #   SQL
+    #
+    def create_continuous_aggregate(name, query, **options)
       execute <<~SQL
         CREATE MATERIALIZED VIEW #{name}
         WITH (timescaledb.continuous) AS
@@ -69,6 +90,15 @@ module Timescale
         SQL
       end
     end
+    alias_method :create_continuous_aggregates, :create_continuous_aggregate
+
+    def create_retention_policy(table_name, interval:)
+      execute "SELECT add_retention_policy('#{table_name}', INTERVAL '#{interval}')"
+    end
+
+    def remove_retention_policy(table_name)
+      execute "SELECT remove_retention_policy('#{table_name}')"
+    end
   end
 end
 

data/lib/timescale/scenic/adapter.rb

@@ -0,0 +1,55 @@
+require 'scenic/adapters/postgres'
+require 'scenic/adapters/postgres/views'
+
+module Timescale
+  module Scenic
+    class Views < ::Scenic::Adapters::Postgres::Views
+      # All of the views that this connection has defined, excluding any
+      # Timescale continuous aggregates. Those should be defined using
+      # +create_continuous_aggregate+ rather than +create_view+.
+      #
+      # @return [Array<Scenic::View>]
+      def all
+        ts_views = views_from_timescale.map { |v| to_scenic_view(v) }
+        pg_views = views_from_postgres.map { |v| to_scenic_view(v) }
+        ts_view_names = ts_views.map(&:name)
+        # Skip records with matching names (includes the schema name
+        # for records not in the public schema)
+        pg_views.reject { |v| v.name.in?(ts_view_names) }
+      end
+
+      private
+
+      def views_from_timescale
+        connection.execute(<<-SQL.squish)
+          SELECT
+            view_name as viewname,
+            view_definition AS definition,
+            'm' AS kind,
+            view_schema AS namespace
+          FROM timescaledb_information.continuous_aggregates
+        SQL
+      end
+    end
+
+    class Adapter < ::Scenic::Adapters::Postgres
+      # Timescale does some funky stuff under the hood with continuous
+      # aggregates. A continuous aggregate is made up of:
+      #
+      # 1. A hypertable to store the materialized data
+      # 2. An entry in the jobs table to refresh the data
+      # 3. A view definition that union's the hypertable and any recent data
+      #    not included in the hypertable
+      #
+      # That doesn't dump well, even to structure.sql (we lose the job
+      # definition, since it's not part of the DDL).
+      #
+      # Our schema dumper implementation will handle dumping the continuous
+      # aggregate definintions, but we need to override Scenic's schema dumping
+      # to exclude those continuous aggregates.
+      def views
+        Views.new(connection).all
+      end
+    end
+  end
+end

data/lib/timescale/schema_dumper.rb

@@ -0,0 +1,88 @@
+require 'active_record/connection_adapters/postgresql_adapter'
+require 'active_support/core_ext/string/indent'
+
+module Timescale
+  module SchemaDumper
+    def tables(stream)
+      super # This will call #table for each table in the database
+      views(stream) unless defined?(Scenic) # Don't call this twice if we're using Scenic
+    end
+
+    def table(table_name, stream)
+      super(table_name, stream)
+      if hypertable = Timescale::Hypertable.find_by(hypertable_name: table_name)
+        timescale_hypertable(hypertable, stream)
+        timescale_retention_policy(hypertable, stream)
+      end
+    end
+
+    def views(stream)
+      timescale_continuous_aggregates(stream) # Define these before any Scenic views that might use them
+      super if defined?(super)
+    end
+
+    private
+
+    def timescale_hypertable(hypertable, stream)
+      dim = hypertable.dimensions
+      extra_settings = {
+        time_column: "#{dim.column_name}",
+        chunk_time_interval: "#{dim.time_interval.inspect}"
+      }.merge(timescale_compression_settings_for(hypertable)).map {|k, v| %Q[#{k}: "#{v}"]}.join(", ")
+
+      stream.puts %Q[  create_hypertable "#{hypertable.hypertable_name}", #{extra_settings}]
+      stream.puts
+    end
+
+    def timescale_retention_policy(hypertable, stream)
+      hypertable.jobs.where(proc_name: "policy_retention").each do |job|
+        stream.puts %Q[  create_retention_policy "#{job.hypertable_name}", interval: "#{job.config["drop_after"]}"]
+        stream.puts
+      end
+    end
+
+    def timescale_compression_settings_for(hypertable)
+      compression_settings = hypertable.compression_settings.each_with_object({}) do |setting, compression_settings|
+        compression_settings[:compress_segmentby] = setting.attname if setting.segmentby_column_index
+
+        if setting.orderby_column_index
+          direction = setting.orderby_asc ? "ASC" : "DESC"
+          compression_settings[:compress_orderby] = "#{setting.attname} #{direction}"
+        end
+      end
+
+      hypertable.jobs.compression.each do |job|
+        compression_settings[:compression_interval] = job.config["compress_after"]
+      end
+      compression_settings
+    end
+
+    def timescale_continuous_aggregates(stream)
+      Timescale::ContinuousAggregates.all.each do |aggregate|
+        opts = if (refresh_policy = aggregate.jobs.refresh_continuous_aggregate.first)
+                 interval = timescale_interval(refresh_policy.schedule_interval)
+                 end_offset = timescale_interval(refresh_policy.config["end_offset"])
+                 start_offset = timescale_interval(refresh_policy.config["start_offset"])
+                 %Q[, refresh_policies: { start_offset: "#{start_offset}", end_offset: "#{end_offset}", schedule_interval: "#{interval}"}]
+               else
+                 ""
+               end
+
+        stream.puts <<~AGG.indent(2)
+          create_continuous_aggregate("#{aggregate.view_name}", <<-SQL#{opts})
+            #{aggregate.view_definition.strip.gsub(/;$/, "")}
+          SQL
+        AGG
+        stream.puts
+      end
+    end
+
+    def timescale_interval(value)
+      return "NULL" if value.nil? || value.to_s.downcase == "null"
+
+      "INTERVAL '#{value}'"
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.prepend(Timescale::SchemaDumper)

data/lib/timescale/stats_report.rb

@@ -0,0 +1,35 @@
+require "active_support/core_ext/numeric/conversions"
+
+module Timescale
+  module StatsReport
+    module_function
+    def resume(scope=Hypertable.all)
+      base_filter = {hypertable_name: scope.pluck(:hypertable_name)}
+      {
+        hypertables: {
+          count: scope.count,
+          uncompressed: scope.to_a.count { |h| h.compression_stats.empty? },
+          approximate_row_count: approximate_row_count(scope),
+          chunks: Chunk.where(base_filter).resume,
+          size: compression_resume(scope)
+        },
+        continuous_aggregates: ContinuousAggregates.where(base_filter).resume,
+        jobs_stats: JobStats.where(base_filter).resume
+      }
+    end
+
+    def compression_resume(scope)
+      sum = -> (method) { (scope.map(&method).inject(:+) || 0).to_s(:human_size)}
+      {
+        uncompressed: sum[:before_total_bytes],
+        compressed: sum[:after_total_bytes]
+      }
+    end
+
+    def approximate_row_count(scope)
+      scope.to_a.map do |hypertable|
+        { hypertable.hypertable_name => hypertable.approximate_row_count }
+      end.inject(&:merge!)
+    end
+  end
+end

data/lib/timescale/version.rb

@@ -1,3 +1,3 @@
 module Timescale
-  VERSION = "0.1.0"
+  VERSION = '0.1.5'
 end

data/lib/timescale.rb

@@ -1,16 +1,22 @@
-require "timescale/version"
 require 'active_record'
+
+require_relative 'timescale/acts_as_hypertable'
+require_relative 'timescale/acts_as_hypertable/core'
 require_relative 'timescale/chunk'
+require_relative 'timescale/compression_settings'
+require_relative 'timescale/continuous_aggregates'
+require_relative 'timescale/dimensions'
 require_relative 'timescale/hypertable'
 require_relative 'timescale/job'
 require_relative 'timescale/job_stats'
-require_relative 'timescale/continuous_aggregates'
-require_relative 'timescale/compression_settings'
-require_relative 'timescale/hypertable_helpers'
+require_relative 'timescale/schema_dumper'
+require_relative 'timescale/stats_report'
 require_relative 'timescale/migration_helpers'
+require_relative 'timescale/version'
 
 module Timescale
   module_function
+
   def chunks
     Chunk.all
   end
@@ -34,4 +40,23 @@ module Timescale
   def job_stats
     JobStats.all
   end
+
+  def stats(scope=Hypertable.all)
+    StatsReport.resume(scope)
+  end
+
+  def default_hypertable_options
+    Timescale::ActsAsHypertable::DEFAULT_OPTIONS
+  end
+end
+
+begin
+  require 'scenic'
+  require_relative 'timescale/scenic/adapter'
+
+  Scenic.configure do |config|
+    config.database = Timescale::Scenic::Adapter.new
+  end
+rescue LoadError
+  # This is expected when the scenic gem is not being used
 end

data/timescale.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Jônatas Davi Paganini"]
   spec.email         = ["jonatasdp@gmail.com"]
 
-  spec.summary       = %q{TimesaleDB helpers for Ruby ecosystem.}
+  spec.summary       = %q{TimescaleDB helpers for Ruby ecosystem.}
   spec.description   = %q{Functions from timescaledb available in the ActiveRecord models.}
   spec.homepage      = "https://github.com/jonatas/timescale"
   spec.license       = "MIT"
@@ -23,10 +23,18 @@ Gem::Specification.new do |spec|
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/tsdb}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency  "pg", "~> 1.2"
-  spec.add_dependency 'activerecord'
+  spec.add_dependency "pg", "~> 1.2"
+  spec.add_dependency "activerecord"
+  spec.add_dependency "activesupport"
+
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rspec-its"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "dotenv"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "database_cleaner-active_record"
 end