timescaledb 0.1.2 → 0.2.0

data/lib/timescaledb/acts_as_hypertable/core.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Timescaledb
+  module ActsAsHypertable
+    module Core
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def time_column
+          @time_column ||= hypertable_options[:time_column] || :created_at
+        end
+
+        protected
+
+        def define_association_scopes
+          scope :chunks, -> do
+            Chunk.where(hypertable_name: table_name)
+          end
+
+          scope :hypertable, -> do
+            Hypertable.find_by(hypertable_name: table_name)
+          end
+
+          scope :jobs, -> do
+            Job.where(hypertable_name: table_name)
+          end
+
+          scope :job_stats, -> do
+            JobStats.where(hypertable_name: table_name)
+          end
+
+          scope :compression_settings, -> do
+            CompressionSettings.where(hypertable_name: table_name)
+          end
+
+          scope :continuous_aggregates, -> do
+            ContinuousAggregates.where(hypertable_name: table_name)
+          end
+        end
+
+        def define_default_scopes
+          scope :previous_month, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.last_month.in_time_zone.beginning_of_month.to_date,
+              end_time: Date.today.last_month.in_time_zone.end_of_month.to_date
+            )
+          end
+
+          scope :previous_week, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.last_week.in_time_zone.beginning_of_week.to_date,
+              end_time: Date.today.last_week.in_time_zone.end_of_week.to_date
+            )
+          end
+
+          scope :this_month, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.in_time_zone.beginning_of_month.to_date,
+              end_time: Date.today.in_time_zone.end_of_month.to_date
+            )
+          end
+
+          scope :this_week, -> do
+            where(
+              "DATE(#{time_column}) >= :start_time AND DATE(#{time_column}) <= :end_time",
+              start_time: Date.today.in_time_zone.beginning_of_week.to_date,
+              end_time: Date.today.in_time_zone.end_of_week.to_date
+            )
+          end
+
+          scope :yesterday, -> { where("DATE(#{time_column}) = ?", Date.yesterday.in_time_zone.to_date) }
+          scope :today, -> { where("DATE(#{time_column}) = ?", Date.today.in_time_zone.to_date) }
+          scope :last_hour, -> { where("#{time_column} > ?", 1.hour.ago.in_time_zone) }
+        end
+
+        def normalize_hypertable_options
+          hypertable_options[:time_column] = hypertable_options[:time_column].to_sym
+        end
+      end
+    end
+  end
+end

data/lib/timescaledb/acts_as_hypertable.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Timescaledb
+  # If you want your model to hook into its underlying hypertable
+  # as well as have access to TimescaleDB specific data, methods, and more,
+  # specify this macro in your model.
+  #
+  # @note Your model's table needs to have already been converted to a hypertable
+  # via the TimescaleDB `create_hypertable` function for this to work.
+  #
+  # @see https://docs.timescale.com/api/latest/hypertable/create_hypertable/ for
+  #   how to use the SQL `create_hypertable` function.
+  # @see Timescaledb::MigrationHelpers#create_table for how to create a new hypertable
+  # via a Rails migration utilizing the standard `create_table` method.
+  #
+  # @example Enabling the macro on your model
+  #   class Event < ActiveRecord::Base
+  #     acts_as_hypertable
+  #   end
+  #
+  # @see Timescaledb::ActsAsHypertable#acts_as_hypertable
+  #   for configuration options
+  module ActsAsHypertable
+    DEFAULT_OPTIONS = {
+      time_column: :created_at
+    }.freeze
+
+    def acts_as_hypertable?
+      included_modules.include?(Timescaledb::ActsAsHypertable::Core)
+    end
+
+    # == Configuration options
+    #
+    # @param [Hash] options The options to initialize your macro with.
+    # @option options [Symbol] :time_column The name of the column in your
+    #   model's table containing time values. The name provided should be
+    #   the same name as the `time_column_name` you passed to the
+    #   TimescaleDB `create_hypertable` function.
+    #
+    # @example Enabling the macro on your model with options
+    #   class Event < ActiveRecord::Base
+    #     acts_as_hypertable time_column: :timestamp
+    #   end
+    #
+    def acts_as_hypertable(options = {})
+      return if acts_as_hypertable?
+
+      include Timescaledb::ActsAsHypertable::Core
+
+      class_attribute :hypertable_options, instance_writer: false
+
+      self.hypertable_options = DEFAULT_OPTIONS.dup
+      hypertable_options.merge!(options)
+      normalize_hypertable_options
+
+      define_association_scopes
+      define_default_scopes
+    end
+  end
+end
+
+ActiveRecord::Base.extend Timescaledb::ActsAsHypertable

data/lib/{timescale → timescaledb}/chunk.rb

@@ -1,4 +1,4 @@
-module Timescale
+module Timescaledb
   class Chunk < ActiveRecord::Base
     self.table_name = "timescaledb_information.chunks"
     self.primary_key = "chunk_name"
@@ -8,6 +8,14 @@ module Timescale
     scope :compressed, -> { where(is_compressed: true) }
     scope :uncompressed, -> { where(is_compressed: false) }
 
+    scope :resume, -> do
+      {
+        total: count,
+        compressed: compressed.count,
+        uncompressed: uncompressed.count
+      }
+    end
+
     def compress!
       execute("SELECT compress_chunk(#{chunk_relation})")
     end

data/lib/{timescale → timescaledb}/compression_settings.rb

@@ -1,6 +1,7 @@
-module Timescale
-  class CompressionSettings < ActiveRecord::Base
+module Timescaledb
+  class CompressionSetting < ActiveRecord::Base
     self.table_name = "timescaledb_information.compression_settings"
     belongs_to :hypertable, foreign_key: :hypertable_name
   end
+  CompressionSettings = CompressionSetting
 end

data/lib/timescaledb/continuous_aggregates.rb

@@ -0,0 +1,19 @@
+module Timescaledb
+  class ContinuousAggregate < ActiveRecord::Base
+    self.table_name = "timescaledb_information.continuous_aggregates"
+    self.primary_key = 'materialization_hypertable_name'
+
+    has_many :jobs, foreign_key: "hypertable_name",
+      class_name: "Timescaledb::Job"
+
+    has_many :chunks, foreign_key: "hypertable_name",
+      class_name: "Timescaledb::Chunk"
+
+    scope :resume, -> do
+      {
+        total: count
+      }
+    end
+  end
+  ContinuousAggregates = ContinuousAggregate
+end

data/lib/timescaledb/dimensions.rb

@@ -0,0 +1,6 @@
+module Timescaledb
+  class Dimension < ActiveRecord::Base
+    self.table_name = "timescaledb_information.dimensions"
+  end
+  Dimensions = Dimension
+end

data/lib/{timescale → timescaledb}/hypertable.rb

@@ -1,7 +1,6 @@
-module Timescale
+module Timescaledb
   class Hypertable < ActiveRecord::Base
     self.table_name = "timescaledb_information.hypertables"
-
     self.primary_key = "hypertable_name"
 
     has_many :jobs, foreign_key: "hypertable_name"
@@ -9,11 +8,15 @@ module Timescale
 
     has_many :compression_settings,
       foreign_key: "hypertable_name",
-      class_name: "Timescale::CompressionSettings"
+      class_name: "Timescaledb::CompressionSetting"
+
+    has_one :dimensions,
+      foreign_key: "hypertable_name",
+      class_name: "Timescaledb::Dimension"
 
     has_many :continuous_aggregates,
       foreign_key: "hypertable_name",
-      class_name: "Timescale::ContinuousAggregates"
+      class_name: "Timescaledb::ContinuousAggregate"
 
     def chunks_detailed_size
       struct_from "SELECT * from chunks_detailed_size('#{self.hypertable_name}')"
@@ -24,7 +27,8 @@ module Timescale
     end
 
     def compression_stats
-      struct_from("SELECT * from hypertable_compression_stats('#{self.hypertable_name}')").first || {}
+      @compression_stats ||=
+        struct_from("SELECT * from hypertable_compression_stats('#{self.hypertable_name}')").first || {}
     end
 
     def detailed_size

data/lib/timescaledb/job.rb

@@ -0,0 +1,10 @@
+module Timescaledb
+  class Job < ActiveRecord::Base
+    self.table_name = "timescaledb_information.jobs"
+    self.primary_key = "job_id"
+
+    scope :compression, -> { where(proc_name: [:tsbs_compress_chunks, :policy_compression]) }
+    scope :refresh_continuous_aggregate, -> { where(proc_name: :policy_refresh_continuous_aggregate) }
+    scope :scheduled, -> { where(scheduled: true) }
+  end
+end

data/lib/{timescale → timescaledb}/job_stats.rb

@@ -1,11 +1,9 @@
-module Timescale
-  class JobStats < ActiveRecord::Base
+module Timescaledb
+  class JobStat < ActiveRecord::Base
     self.table_name = "timescaledb_information.job_stats"
 
     belongs_to :job
 
-    attribute :last_run_duration, :interval
-
     scope :success, -> { where(last_run_status: "Success") }
     scope :scheduled, -> { where(job_status: "Scheduled") }
     scope :resume, -> do
@@ -15,4 +13,5 @@ module Timescale
         .to_a.map{|e|e.attributes.transform_keys(&:to_sym) }
     end
   end
+  JobStats = JobStat
 end

data/lib/{timescale → timescaledb}/migration_helpers.rb

@@ -1,7 +1,7 @@
 require 'active_record/connection_adapters/postgresql_adapter'
 
 # Useful methods to run TimescaleDB in you Ruby app.
-module Timescale
+module Timescaledb
   # Migration helpers can help you to setup hypertables by default.
   module MigrationHelpers
     # create_table can receive `hypertable` argument
@@ -21,12 +21,12 @@ module Timescale
     #  end
     def create_table(table_name, id: :primary_key, primary_key: nil, force: nil, **options)
       super
-      setup_hypertable_options(table_name, **options[:hypertable]) if options.key?(:hypertable)
+      create_hypertable(table_name, **options[:hypertable]) if options.key?(:hypertable)
     end
 
     # Setup hypertable from options
     # @see create_table with the hypertable options.
-    def setup_hypertable_options(table_name,
+    def create_hypertable(table_name,
                                  time_column: 'created_at',
                                  chunk_time_interval: '1 week',
                                  compress_segmentby: nil,
@@ -51,7 +51,28 @@ module Timescale
       end
     end
 
-    def create_continuous_aggregates(name, query, **options)
+    # Create a new continuous aggregate
+    #
+    # @param name [String, Symbol] The name of the continuous aggregate.
+    # @param query [String] The SQL query for the aggregate view definition.
+    # @param with_data [Boolean] Set to true to create the aggregate WITH DATA
+    # @param refresh_policies [Hash] Set to create a refresh policy
+    # @option refresh_policies [String] start_offset: INTERVAL or integer
+    # @option refresh_policies [String] end_offset: INTERVAL or integer
+    # @option refresh_policies [String] schedule_interval: INTERVAL
+    #
+    # @see https://docs.timescale.com/api/latest/continuous-aggregates/add_continuous_aggregate_policy/
+    #
+    # @example
+    #   create_continuous_aggregate(:activity_counts, query: <<-SQL, refresh_policies: { schedule_interval: "INTERVAL '1 hour'" })
+    #     SELECT
+    #       time_bucket(INTERVAL '1 day', activity.created_at) AS bucket,
+    #       count(*)
+    #     FROM activity
+    #     GROUP BY bucket
+    #   SQL
+    #
+    def create_continuous_aggregate(name, query, **options)
       execute <<~SQL
         CREATE MATERIALIZED VIEW #{name}
         WITH (timescaledb.continuous) AS
@@ -69,7 +90,16 @@ module Timescale
         SQL
       end
     end
+    alias_method :create_continuous_aggregates, :create_continuous_aggregate
+
+    def create_retention_policy(table_name, interval:)
+      execute "SELECT add_retention_policy('#{table_name}', INTERVAL '#{interval}')"
+    end
+
+    def remove_retention_policy(table_name)
+      execute "SELECT remove_retention_policy('#{table_name}')"
+    end
   end
 end
 
-ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.include(Timescale::MigrationHelpers)
+ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.include(Timescaledb::MigrationHelpers)

data/lib/timescaledb/scenic/adapter.rb

@@ -0,0 +1,55 @@
+require 'scenic/adapters/postgres'
+require 'scenic/adapters/postgres/views'
+
+module Timescaledb
+  module Scenic
+    class Views < ::Scenic::Adapters::Postgres::Views
+      # All of the views that this connection has defined, excluding any
+      # Timescale continuous aggregates. Those should be defined using
+      # +create_continuous_aggregate+ rather than +create_view+.
+      #
+      # @return [Array<Scenic::View>]
+      def all
+        ts_views = views_from_timescale.map { |v| to_scenic_view(v) }
+        pg_views = views_from_postgres.map { |v| to_scenic_view(v) }
+        ts_view_names = ts_views.map(&:name)
+        # Skip records with matching names (includes the schema name
+        # for records not in the public schema)
+        pg_views.reject { |v| v.name.in?(ts_view_names) }
+      end
+
+      private
+
+      def views_from_timescale
+        connection.execute(<<-SQL.squish)
+          SELECT
+            view_name as viewname,
+            view_definition AS definition,
+            'm' AS kind,
+            view_schema AS namespace
+          FROM timescaledb_information.continuous_aggregates
+        SQL
+      end
+    end
+
+    class Adapter < ::Scenic::Adapters::Postgres
+      # Timescale does some funky stuff under the hood with continuous
+      # aggregates. A continuous aggregate is made up of:
+      #
+      # 1. A hypertable to store the materialized data
+      # 2. An entry in the jobs table to refresh the data
+      # 3. A view definition that union's the hypertable and any recent data
+      #    not included in the hypertable
+      #
+      # That doesn't dump well, even to structure.sql (we lose the job
+      # definition, since it's not part of the DDL).
+      #
+      # Our schema dumper implementation will handle dumping the continuous
+      # aggregate definitions, but we need to override Scenic's schema dumping
+      # to exclude those continuous aggregates.
+      def views
+        Views.new(connection).all
+      end
+    end
+  end
+end

data/lib/timescaledb/scenic/extension.rb

@@ -0,0 +1,72 @@
+# Scenic does not include `WITH` option that is used with continuous aggregates.
+module Timescaledb
+  module Scenic
+    module Extension
+      # @override Scenic::Adapters::Postgres#create_materialized_view
+      # Creates a materialized view in the database
+      #
+      # @param name The name of the materialized view to create
+      # @param sql_definition The SQL schema that defines the materialized view.
+      # @param with [String] Default: nil. Set with: "..." to add "WITH (...)".
+      # @param no_data [Boolean] Default: false. Set to true to not create data.
+      #   materialized view without running the associated query. You will need
+      #   to perform a non-concurrent refresh to populate with data.
+      #
+      # This is typically called in a migration via {Statements#create_view}.
+      # @return [void]
+      def create_materialized_view(name, sql_definition, with: nil, no_data: false)
+        execute <<-SQL
+  CREATE MATERIALIZED VIEW #{quote_table_name(name)}
+  #{"WITH (#{with})" if with} AS
+  #{sql_definition.rstrip.chomp(';')}
+  #{'WITH NO DATA' if no_data};
+        SQL
+      end
+
+      # @override Scenic::Adapters::Postgres#create_view
+      # to add the `with: ` keyword that can be used for such option.
+      #
+      def create_view(name, version: nil, with: nil, sql_definition: nil, materialized: false, no_data: false)
+        if version.present? && sql_definition.present?
+          raise(
+            ArgumentError,
+            "sql_definition and version cannot both be set",
+          )
+        end
+
+        if version.blank? && sql_definition.blank?
+          version = 1
+        end
+
+        sql_definition ||= definition(name, version)
+
+        if materialized
+          ::Scenic.database.create_materialized_view(
+            name,
+            sql_definition,
+            no_data: no_data,
+            with: with
+          )
+        else
+          ::Scenic.database.create_view(name, sql_definition, with: with)
+        end
+      end
+
+      private
+
+      def definition(name, version)
+        ::Scenic::Definition.new(name, version).to_sql
+      end
+    end
+    module MigrationHelpers
+      # Create a timescale continuous aggregate view
+      def create_scenic_continuous_aggregate(name)
+        ::Scenic.database.create_view(name, materialized: true, no_data: true, with: "timescaledb.continuous")
+      end
+    end
+  end
+end
+
+
+ActiveRecord::ConnectionAdapters::AbstractAdapter.include(Timescaledb::Scenic::MigrationHelpers)
+Scenic::Adapters::Postgres.prepend(Timescaledb::Scenic::Extension)

data/lib/timescaledb/schema_dumper.rb

@@ -0,0 +1,88 @@
+require 'active_record/connection_adapters/postgresql_adapter'
+require 'active_support/core_ext/string/indent'
+
+module Timescaledb
+  module SchemaDumper
+    def tables(stream)
+      super # This will call #table for each table in the database
+      views(stream) unless defined?(Scenic) # Don't call this twice if we're using Scenic
+    end
+
+    def table(table_name, stream)
+      super(table_name, stream)
+      if hypertable = Timescaledb::Hypertable.find_by(hypertable_name: table_name)
+        timescale_hypertable(hypertable, stream)
+        timescale_retention_policy(hypertable, stream)
+      end
+    end
+
+    def views(stream)
+      timescale_continuous_aggregates(stream) # Define these before any Scenic views that might use them
+      super if defined?(super)
+    end
+
+    private
+
+    def timescale_hypertable(hypertable, stream)
+      dim = hypertable.dimensions
+      extra_settings = {
+        time_column: "#{dim.column_name}",
+        chunk_time_interval: "#{dim.time_interval.inspect}"
+      }.merge(timescale_compression_settings_for(hypertable)).map {|k, v| %Q[#{k}: "#{v}"]}.join(", ")
+
+      stream.puts %Q[  create_hypertable "#{hypertable.hypertable_name}", #{extra_settings}]
+      stream.puts
+    end
+
+    def timescale_retention_policy(hypertable, stream)
+      hypertable.jobs.where(proc_name: "policy_retention").each do |job|
+        stream.puts %Q[  create_retention_policy "#{job.hypertable_name}", interval: "#{job.config["drop_after"]}"]
+        stream.puts
+      end
+    end
+
+    def timescale_compression_settings_for(hypertable)
+      compression_settings = hypertable.compression_settings.each_with_object({}) do |setting, compression_settings|
+        compression_settings[:compress_segmentby] = setting.attname if setting.segmentby_column_index
+
+        if setting.orderby_column_index
+          direction = setting.orderby_asc ? "ASC" : "DESC"
+          compression_settings[:compress_orderby] = "#{setting.attname} #{direction}"
+        end
+      end
+
+      hypertable.jobs.compression.each do |job|
+        compression_settings[:compression_interval] = job.config["compress_after"]
+      end
+      compression_settings
+    end
+
+    def timescale_continuous_aggregates(stream)
+      Timescaledb::ContinuousAggregates.all.each do |aggregate|
+        opts = if (refresh_policy = aggregate.jobs.refresh_continuous_aggregate.first)
+                 interval = timescale_interval(refresh_policy.schedule_interval)
+                 end_offset = timescale_interval(refresh_policy.config["end_offset"])
+                 start_offset = timescale_interval(refresh_policy.config["start_offset"])
+                 %Q[, refresh_policies: { start_offset: "#{start_offset}", end_offset: "#{end_offset}", schedule_interval: "#{interval}"}]
+               else
+                 ""
+               end
+
+        stream.puts <<~AGG.indent(2)
+          create_continuous_aggregate("#{aggregate.view_name}", <<-SQL#{opts})
+            #{aggregate.view_definition.strip.gsub(/;$/, "")}
+          SQL
+        AGG
+        stream.puts
+      end
+    end
+
+    def timescale_interval(value)
+      return "NULL" if value.nil? || value.to_s.downcase == "null"
+
+      "INTERVAL '#{value}'"
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.prepend(Timescaledb::SchemaDumper)

data/lib/timescaledb/stats_report.rb

@@ -0,0 +1,35 @@
+require "active_support/core_ext/numeric/conversions"
+
+module Timescaledb
+  module StatsReport
+    module_function
+    def resume(scope=Hypertable.all)
+      base_filter = {hypertable_name: scope.pluck(:hypertable_name)}
+      {
+        hypertables: {
+          count: scope.count,
+          uncompressed: scope.to_a.count { |h| h.compression_stats.empty? },
+          approximate_row_count: approximate_row_count(scope),
+          chunks: Chunk.where(base_filter).resume,
+          size: compression_resume(scope)
+        },
+        continuous_aggregates: ContinuousAggregates.where(base_filter).resume,
+        jobs_stats: JobStats.where(base_filter).resume
+      }
+    end
+
+    def compression_resume(scope)
+      sum = -> (method) { (scope.map(&method).inject(:+) || 0).to_formatted_s(:human_size)}
+      {
+        uncompressed: sum[:before_total_bytes],
+        compressed: sum[:after_total_bytes]
+      }
+    end
+
+    def approximate_row_count(scope)
+      scope.to_a.map do |hypertable|
+        { hypertable.hypertable_name => hypertable.approximate_row_count }
+      end.inject(&:merge!)
+    end
+  end
+end

data/lib/timescaledb/version.rb

@@ -0,0 +1,3 @@
+module Timescaledb
+  VERSION = '0.2.0'
+end

data/lib/timescaledb.rb

@@ -0,0 +1,64 @@
+require 'active_record'
+
+require_relative 'timescaledb/acts_as_hypertable'
+require_relative 'timescaledb/acts_as_hypertable/core'
+require_relative 'timescaledb/chunk'
+require_relative 'timescaledb/compression_settings'
+require_relative 'timescaledb/continuous_aggregates'
+require_relative 'timescaledb/dimensions'
+require_relative 'timescaledb/hypertable'
+require_relative 'timescaledb/job'
+require_relative 'timescaledb/job_stats'
+require_relative 'timescaledb/schema_dumper'
+require_relative 'timescaledb/stats_report'
+require_relative 'timescaledb/migration_helpers'
+require_relative 'timescaledb/version'
+
+module Timescaledb
+  module_function
+
+  def chunks
+    Chunk.all
+  end
+
+  def hypertables
+    Hypertable.all
+  end
+
+  def continuous_aggregates
+    ContinuousAggregates.all
+  end
+
+  def compression_settings
+    CompressionSettings.all
+  end
+
+  def jobs
+    Job.all
+  end
+
+  def job_stats
+    JobStats.all
+  end
+
+  def stats(scope=Hypertable.all)
+    StatsReport.resume(scope)
+  end
+
+  def default_hypertable_options
+    Timescaledb::ActsAsHypertable::DEFAULT_OPTIONS
+  end
+end
+
+begin
+  require 'scenic'
+  require_relative 'timescaledb/scenic/adapter'
+  require_relative 'timescaledb/scenic/extension'
+
+  Scenic.configure do |config|
+    config.database = Timescaledb::Scenic::Adapter.new
+  end
+
+rescue LoadError
+  # This is expected when the scenic gem is not being used
+end