timescaledb 0.1.4 → 0.2.1

data/lib/{timescale → timescaledb}/job_stats.rb

@@ -1,11 +1,9 @@
-module Timescale
-  class JobStats < ActiveRecord::Base
+module Timescaledb
+  class JobStat < ActiveRecord::Base
     self.table_name = "timescaledb_information.job_stats"
 
     belongs_to :job
 
-    attribute :last_run_duration, :interval
-
     scope :success, -> { where(last_run_status: "Success") }
     scope :scheduled, -> { where(job_status: "Scheduled") }
     scope :resume, -> do
@@ -15,4 +13,5 @@ module Timescale
         .to_a.map{|e|e.attributes.transform_keys(&:to_sym) }
     end
   end
+  JobStats = JobStat
 end

data/lib/{timescale → timescaledb}/migration_helpers.rb

@@ -1,7 +1,7 @@
 require 'active_record/connection_adapters/postgresql_adapter'
 
 # Useful methods to run TimescaleDB in you Ruby app.
-module Timescale
+module Timescaledb
   # Migration helpers can help you to setup hypertables by default.
   module MigrationHelpers
     # create_table can receive `hypertable` argument
@@ -51,25 +51,63 @@ module Timescale
       end
     end
 
-    def create_continuous_aggregates(name, query, **options)
+    # Create a new continuous aggregate
+    #
+    # @param name [String, Symbol] The name of the continuous aggregate.
+    # @param query [String] The SQL query for the aggregate view definition.
+    # @param with_data [Boolean] Set to true to create the aggregate WITH DATA
+    # @param refresh_policies [Hash] Set to create a refresh policy
+    # @option refresh_policies [String] start_offset: INTERVAL or integer
+    # @option refresh_policies [String] end_offset: INTERVAL or integer
+    # @option refresh_policies [String] schedule_interval: INTERVAL
+    #
+    # @see https://docs.timescale.com/api/latest/continuous-aggregates/add_continuous_aggregate_policy/
+    #
+    # @example
+    #   create_continuous_aggregate(:activity_counts, query: <<-SQL, refresh_policies: { schedule_interval: "INTERVAL '1 hour'" })
+    #     SELECT
+    #       time_bucket(INTERVAL '1 day', activity.created_at) AS bucket,
+    #       count(*)
+    #     FROM activity
+    #     GROUP BY bucket
+    #   SQL
+    #
+    def create_continuous_aggregate(table_name, query, **options)
       execute <<~SQL
-        CREATE MATERIALIZED VIEW #{name}
+        CREATE MATERIALIZED VIEW #{table_name}
         WITH (timescaledb.continuous) AS
         #{query.respond_to?(:to_sql) ? query.to_sql : query}
         WITH #{"NO" unless options[:with_data]} DATA;
       SQL
 
-      if (policy = options[:refresh_policies])
-        # TODO: assert valid keys
-        execute <<~SQL
-          SELECT add_continuous_aggregate_policy('#{name}',
-            start_offset => #{policy[:start_offset]},
-            end_offset => #{policy[:end_offset]},
-            schedule_interval => #{policy[:schedule_interval]});
-        SQL
-      end
+      create_continuous_aggregate_policy(table_name, options[:refresh_policies] || {})
+    end
+    alias_method :create_continuous_aggregates, :create_continuous_aggregate
+
+    def create_continuous_aggregate_policy(table_name, **options)
+      return if options.empty?
+
+      # TODO: assert valid keys
+      execute <<~SQL
+        SELECT add_continuous_aggregate_policy('#{table_name}',
+          start_offset => #{options[:start_offset]},
+          end_offset => #{options[:end_offset]},
+          schedule_interval => #{options[:schedule_interval]});
+      SQL
+    end
+
+    def remove_continuous_aggregate_policy(table_name)
+      execute "SELECT remove_continuous_aggregate_policy('#{table_name}')"
+    end
+
+    def create_retention_policy(table_name, interval:)
+      execute "SELECT add_retention_policy('#{table_name}', INTERVAL '#{interval}')"
+    end
+
+    def remove_retention_policy(table_name)
+      execute "SELECT remove_retention_policy('#{table_name}')"
     end
   end
 end
 
-ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.include(Timescale::MigrationHelpers)
+ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.include(Timescaledb::MigrationHelpers)

data/lib/timescaledb/scenic/adapter.rb

@@ -0,0 +1,55 @@
+require 'scenic/adapters/postgres'
+require 'scenic/adapters/postgres/views'
+
+module Timescaledb
+  module Scenic
+    class Views < ::Scenic::Adapters::Postgres::Views
+      # All of the views that this connection has defined, excluding any
+      # Timescale continuous aggregates. Those should be defined using
+      # +create_continuous_aggregate+ rather than +create_view+.
+      #
+      # @return [Array<Scenic::View>]
+      def all
+        ts_views = views_from_timescale.map { |v| to_scenic_view(v) }
+        pg_views = views_from_postgres.map { |v| to_scenic_view(v) }
+        ts_view_names = ts_views.map(&:name)
+        # Skip records with matching names (includes the schema name
+        # for records not in the public schema)
+        pg_views.reject { |v| v.name.in?(ts_view_names) }
+      end
+
+      private
+
+      def views_from_timescale
+        connection.execute(<<-SQL.squish)
+          SELECT
+            view_name as viewname,
+            view_definition AS definition,
+            'm' AS kind,
+            view_schema AS namespace
+          FROM timescaledb_information.continuous_aggregates
+        SQL
+      end
+    end
+
+    class Adapter < ::Scenic::Adapters::Postgres
+      # Timescale does some funky stuff under the hood with continuous
+      # aggregates. A continuous aggregate is made up of:
+      #
+      # 1. A hypertable to store the materialized data
+      # 2. An entry in the jobs table to refresh the data
+      # 3. A view definition that union's the hypertable and any recent data
+      #    not included in the hypertable
+      #
+      # That doesn't dump well, even to structure.sql (we lose the job
+      # definition, since it's not part of the DDL).
+      #
+      # Our schema dumper implementation will handle dumping the continuous
+      # aggregate definitions, but we need to override Scenic's schema dumping
+      # to exclude those continuous aggregates.
+      def views
+        Views.new(connection).all
+      end
+    end
+  end
+end

data/lib/timescaledb/scenic/extension.rb

@@ -0,0 +1,72 @@
+# Scenic does not include `WITH` option that is used with continuous aggregates.
+module Timescaledb
+  module Scenic
+    module Extension
+      # @override Scenic::Adapters::Postgres#create_materialized_view
+      # Creates a materialized view in the database
+      #
+      # @param name The name of the materialized view to create
+      # @param sql_definition The SQL schema that defines the materialized view.
+      # @param with [String] Default: nil. Set with: "..." to add "WITH (...)".
+      # @param no_data [Boolean] Default: false. Set to true to not create data.
+      #   materialized view without running the associated query. You will need
+      #   to perform a non-concurrent refresh to populate with data.
+      #
+      # This is typically called in a migration via {Statements#create_view}.
+      # @return [void]
+      def create_materialized_view(name, sql_definition, with: nil, no_data: false)
+        execute <<-SQL
+  CREATE MATERIALIZED VIEW #{quote_table_name(name)}
+  #{"WITH (#{with})" if with} AS
+  #{sql_definition.rstrip.chomp(';')}
+  #{'WITH NO DATA' if no_data};
+        SQL
+      end
+
+      # @override Scenic::Adapters::Postgres#create_view
+      # to add the `with: ` keyword that can be used for such option.
+      #
+      def create_view(name, version: nil, with: nil, sql_definition: nil, materialized: false, no_data: false)
+        if version.present? && sql_definition.present?
+          raise(
+            ArgumentError,
+            "sql_definition and version cannot both be set",
+          )
+        end
+
+        if version.blank? && sql_definition.blank?
+          version = 1
+        end
+
+        sql_definition ||= definition(name, version)
+
+        if materialized
+          ::Scenic.database.create_materialized_view(
+            name,
+            sql_definition,
+            no_data: no_data,
+            with: with
+          )
+        else
+          ::Scenic.database.create_view(name, sql_definition, with: with)
+        end
+      end
+
+      private
+
+      def definition(name, version)
+        ::Scenic::Definition.new(name, version).to_sql
+      end
+    end
+    module MigrationHelpers
+      # Create a timescale continuous aggregate view
+      def create_scenic_continuous_aggregate(name)
+        ::Scenic.database.create_view(name, materialized: true, no_data: true, with: "timescaledb.continuous")
+      end
+    end
+  end
+end
+
+
+ActiveRecord::ConnectionAdapters::AbstractAdapter.include(Timescaledb::Scenic::MigrationHelpers)
+Scenic::Adapters::Postgres.prepend(Timescaledb::Scenic::Extension)

data/lib/timescaledb/schema_dumper.rb

@@ -0,0 +1,91 @@
+require 'active_record/connection_adapters/postgresql_adapter'
+require 'active_support/core_ext/string/indent'
+
+module Timescaledb
+  module SchemaDumper
+    def tables(stream)
+      super # This will call #table for each table in the database
+      views(stream) unless defined?(Scenic) # Don't call this twice if we're using Scenic
+    end
+
+    def table(table_name, stream)
+      super(table_name, stream)
+      if Timescaledb::Hypertable.table_exists? &&
+         (hypertable = Timescaledb::Hypertable.find_by(hypertable_name: table_name))
+        timescale_hypertable(hypertable, stream)
+        timescale_retention_policy(hypertable, stream)
+      end
+    end
+
+    def views(stream)
+      return unless Timescaledb::ContinuousAggregates.table_exists?
+
+      timescale_continuous_aggregates(stream) # Define these before any Scenic views that might use them
+      super if defined?(super)
+    end
+
+    private
+
+    def timescale_hypertable(hypertable, stream)
+      dim = hypertable.dimensions
+      extra_settings = {
+        time_column: "#{dim.column_name}",
+        chunk_time_interval: "#{dim.time_interval.inspect}"
+      }.merge(timescale_compression_settings_for(hypertable)).map {|k, v| %Q[#{k}: "#{v}"]}.join(", ")
+
+      stream.puts %Q[  create_hypertable "#{hypertable.hypertable_name}", #{extra_settings}]
+      stream.puts
+    end
+
+    def timescale_retention_policy(hypertable, stream)
+      hypertable.jobs.where(proc_name: "policy_retention").each do |job|
+        stream.puts %Q[  create_retention_policy "#{job.hypertable_name}", interval: "#{job.config["drop_after"]}"]
+        stream.puts
+      end
+    end
+
+    def timescale_compression_settings_for(hypertable)
+      compression_settings = hypertable.compression_settings.each_with_object({}) do |setting, compression_settings|
+        compression_settings[:compress_segmentby] = setting.attname if setting.segmentby_column_index
+
+        if setting.orderby_column_index
+          direction = setting.orderby_asc ? "ASC" : "DESC"
+          compression_settings[:compress_orderby] = "#{setting.attname} #{direction}"
+        end
+      end
+
+      hypertable.jobs.compression.each do |job|
+        compression_settings[:compression_interval] = job.config["compress_after"]
+      end
+      compression_settings
+    end
+
+    def timescale_continuous_aggregates(stream)
+      Timescaledb::ContinuousAggregates.all.each do |aggregate|
+        opts = if (refresh_policy = aggregate.jobs.refresh_continuous_aggregate.first)
+                 interval = timescale_interval(refresh_policy.schedule_interval)
+                 end_offset = timescale_interval(refresh_policy.config["end_offset"])
+                 start_offset = timescale_interval(refresh_policy.config["start_offset"])
+                 %Q[, refresh_policies: { start_offset: "#{start_offset}", end_offset: "#{end_offset}", schedule_interval: "#{interval}"}]
+               else
+                 ""
+               end
+
+        stream.puts <<~AGG.indent(2)
+          create_continuous_aggregate("#{aggregate.view_name}", <<-SQL#{opts})
+            #{aggregate.view_definition.strip.gsub(/;$/, "")}
+          SQL
+        AGG
+        stream.puts
+      end
+    end
+
+    def timescale_interval(value)
+      return "NULL" if value.nil? || value.to_s.downcase == "null"
+
+      "INTERVAL '#{value}'"
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.prepend(Timescaledb::SchemaDumper)

data/lib/{timescale → timescaledb}/stats_report.rb

@@ -1,6 +1,6 @@
 require "active_support/core_ext/numeric/conversions"
 
-module Timescale
+module Timescaledb
   module StatsReport
     module_function
     def resume(scope=Hypertable.all)
@@ -19,7 +19,7 @@ module Timescale
     end
 
     def compression_resume(scope)
-      sum = -> (method) { (scope.map(&method).inject(:+) || 0).to_s(:human_size)}
+      sum = -> (method) { (scope.map(&method).inject(:+) || 0).to_formatted_s(:human_size)}
       {
         uncompressed: sum[:before_total_bytes],
         compressed: sum[:after_total_bytes]

data/lib/timescaledb/version.rb

@@ -0,0 +1,3 @@
+module Timescaledb
+  VERSION = '0.2.1'
+end

data/lib/timescaledb.rb

@@ -0,0 +1,64 @@
+require 'active_record'
+
+require_relative 'timescaledb/acts_as_hypertable'
+require_relative 'timescaledb/acts_as_hypertable/core'
+require_relative 'timescaledb/chunk'
+require_relative 'timescaledb/compression_settings'
+require_relative 'timescaledb/continuous_aggregates'
+require_relative 'timescaledb/dimensions'
+require_relative 'timescaledb/hypertable'
+require_relative 'timescaledb/job'
+require_relative 'timescaledb/job_stats'
+require_relative 'timescaledb/schema_dumper'
+require_relative 'timescaledb/stats_report'
+require_relative 'timescaledb/migration_helpers'
+require_relative 'timescaledb/version'
+
+module Timescaledb
+  module_function
+
+  def chunks
+    Chunk.all
+  end
+
+  def hypertables
+    Hypertable.all
+  end
+
+  def continuous_aggregates
+    ContinuousAggregates.all
+  end
+
+  def compression_settings
+    CompressionSettings.all
+  end
+
+  def jobs
+    Job.all
+  end
+
+  def job_stats
+    JobStats.all
+  end
+
+  def stats(scope=Hypertable.all)
+    StatsReport.resume(scope)
+  end
+
+  def default_hypertable_options
+    Timescaledb::ActsAsHypertable::DEFAULT_OPTIONS
+  end
+end
+
+begin
+  require 'scenic'
+  require_relative 'timescaledb/scenic/adapter'
+  require_relative 'timescaledb/scenic/extension'
+
+  Scenic.configure do |config|
+    config.database = Timescaledb::Scenic::Adapter.new
+  end
+
+rescue LoadError
+  # This is expected when the scenic gem is not being used
+end

data/{timescale.gemspec → timescaledb.gemspec}

@@ -1,14 +1,14 @@
-require_relative 'lib/timescale/version'
+require_relative 'lib/timescaledb/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "timescaledb"
-  spec.version       = Timescale::VERSION
+  spec.version       = Timescaledb::VERSION
   spec.authors       = ["Jônatas Davi Paganini"]
   spec.email         = ["jonatasdp@gmail.com"]
 
   spec.summary       = %q{TimescaleDB helpers for Ruby ecosystem.}
   spec.description   = %q{Functions from timescaledb available in the ActiveRecord models.}
-  spec.homepage      = "https://github.com/jonatas/timescale"
+  spec.homepage      = "https://github.com/jonatas/timescaledb"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
@@ -27,8 +27,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/tsdb}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency  "pg", "~> 1.2"
+  spec.add_dependency "pg", "~> 1.2"
   spec.add_dependency "activerecord"
+  spec.add_dependency "activesupport"
 
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rspec-its"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timescaledb
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.1
 platform: ruby
 authors:
 - Jônatas Davi Paganini
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-08 00:00:00.000000000 Z
+date: 2022-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -132,40 +146,95 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
+- ".tool-versions"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
+- Gemfile.scenic
+- Gemfile.scenic.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
 - bin/tsdb
-- examples/Gemfile
-- examples/Gemfile.lock
-- examples/all_in_one.rb
-- lib/timescale.rb
-- lib/timescale/acts_as_hypertable.rb
-- lib/timescale/acts_as_hypertable/core.rb
-- lib/timescale/chunk.rb
-- lib/timescale/compression_settings.rb
-- lib/timescale/continuous_aggregates.rb
-- lib/timescale/dimensions.rb
-- lib/timescale/hypertable.rb
-- lib/timescale/job.rb
-- lib/timescale/job_stats.rb
-- lib/timescale/migration_helpers.rb
-- lib/timescale/schema_dumper.rb
-- lib/timescale/stats_report.rb
-- lib/timescale/version.rb
-- timescale.gemspec
-homepage: https://github.com/jonatas/timescale
+- examples/all_in_one/Gemfile
+- examples/all_in_one/Gemfile.lock
+- examples/all_in_one/all_in_one.rb
+- examples/ranking/.gitattributes
+- examples/ranking/.gitignore
+- examples/ranking/.ruby-version
+- examples/ranking/Gemfile
+- examples/ranking/Gemfile.lock
+- examples/ranking/README.md
+- examples/ranking/Rakefile
+- examples/ranking/app/controllers/application_controller.rb
+- examples/ranking/app/controllers/concerns/.keep
+- examples/ranking/app/jobs/application_job.rb
+- examples/ranking/app/models/application_record.rb
+- examples/ranking/app/models/concerns/.keep
+- examples/ranking/app/models/game.rb
+- examples/ranking/app/models/play.rb
+- examples/ranking/bin/bundle
+- examples/ranking/bin/rails
+- examples/ranking/bin/rake
+- examples/ranking/bin/setup
+- examples/ranking/config.ru
+- examples/ranking/config/application.rb
+- examples/ranking/config/boot.rb
+- examples/ranking/config/credentials.yml.enc
+- examples/ranking/config/database.yml
+- examples/ranking/config/environment.rb
+- examples/ranking/config/environments/development.rb
+- examples/ranking/config/environments/production.rb
+- examples/ranking/config/environments/test.rb
+- examples/ranking/config/initializers/cors.rb
+- examples/ranking/config/initializers/filter_parameter_logging.rb
+- examples/ranking/config/initializers/inflections.rb
+- examples/ranking/config/initializers/timescale.rb
+- examples/ranking/config/locales/en.yml
+- examples/ranking/config/puma.rb
+- examples/ranking/config/routes.rb
+- examples/ranking/config/storage.yml
+- examples/ranking/db/migrate/20220209120747_create_games.rb
+- examples/ranking/db/migrate/20220209120910_create_plays.rb
+- examples/ranking/db/migrate/20220209143347_create_score_per_hours.rb
+- examples/ranking/db/schema.rb
+- examples/ranking/db/seeds.rb
+- examples/ranking/db/views/score_per_hours_v01.sql
+- examples/ranking/lib/tasks/.keep
+- examples/ranking/log/.keep
+- examples/ranking/public/robots.txt
+- examples/ranking/storage/.keep
+- examples/ranking/tmp/.keep
+- examples/ranking/tmp/pids/.keep
+- examples/ranking/tmp/storage/.keep
+- examples/ranking/vendor/.keep
+- lib/timescaledb.rb
+- lib/timescaledb/acts_as_hypertable.rb
+- lib/timescaledb/acts_as_hypertable/core.rb
+- lib/timescaledb/chunk.rb
+- lib/timescaledb/compression_settings.rb
+- lib/timescaledb/continuous_aggregates.rb
+- lib/timescaledb/dimensions.rb
+- lib/timescaledb/hypertable.rb
+- lib/timescaledb/job.rb
+- lib/timescaledb/job_stats.rb
+- lib/timescaledb/migration_helpers.rb
+- lib/timescaledb/scenic/adapter.rb
+- lib/timescaledb/scenic/extension.rb
+- lib/timescaledb/schema_dumper.rb
+- lib/timescaledb/stats_report.rb
+- lib/timescaledb/version.rb
+- timescaledb.gemspec
+homepage: https://github.com/jonatas/timescaledb
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  homepage_uri: https://github.com/jonatas/timescale
+  homepage_uri: https://github.com/jonatas/timescaledb
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -181,7 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: TimescaleDB helpers for Ruby ecosystem.

data/lib/timescale/dimensions.rb

@@ -1,7 +0,0 @@
-module Timescale
-  class Dimensions < ActiveRecord::Base
-    self.table_name = "timescaledb_information.dimensions"
-
-    attribute :time_interval, :interval
-  end
-end

data/lib/timescale/job.rb

@@ -1,13 +0,0 @@
-module Timescale
-  class Job < ActiveRecord::Base
-    self.table_name = "timescaledb_information.jobs"
-    self.primary_key = "job_id"
-
-    attribute :schedule_interval, :interval
-    attribute :max_runtime, :interval
-    attribute :retry_period, :interval
-
-    scope :compression, -> { where(proc_name: "tsbs_compress_chunks") }
-    scope :scheduled, -> { where(scheduled: true) }
-  end
-end

data/lib/timescale/schema_dumper.rb

@@ -1,24 +0,0 @@
-ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.class_eval do
-  def table(table_name, stream)
-    super(table_name, stream)
-    if hypertable=Timescale::Hypertable.find_by(hypertable_name: table_name)
-      dim = hypertable.dimensions
-      # TODO Build compression settings for the template:
-      # #{build_compression_settings_for(hypertable)})
-      stream.puts <<TEMPLATE
-  create_hypertable('#{table_name}',
-                    time_column: '#{dim.column_name}',
-                    chunk_time_interval: '#{dim.time_interval.inspect}')
-TEMPLATE
-    end
-  end
-end
-
-=begin
-    def build_compression_settings_for(hypertable)
-      return if hypertable.compression_settings.nil?
-      hypertable.compression_settings.map do |settings|
-        ", compress_segmentby: #{settings.segmentby_column_index},
-                    compress_orderby: 'created_at',
-                    compression_interval: nil)
-=end

data/lib/timescale/version.rb

@@ -1,3 +0,0 @@
-module Timescale
-  VERSION = '0.1.4'
-end