timescaledb 0.2.6 → 0.2.8

data/lib/timescaledb/schema_dumper.rb

@@ -2,10 +2,14 @@ require 'active_record/connection_adapters/postgresql_adapter'
 require 'active_support/core_ext/string/indent'
 
 module Timescaledb
+  # Schema dumper overrides default schema dumper to include:
+  # * hypertables
+  # * retention policies
+  # * continuous aggregates
+  # * compression settings
   module SchemaDumper
     def tables(stream)
       super # This will call #table for each table in the database
-
       return unless Timescaledb::Hypertable.table_exists?
 
       timescale_hypertables(stream)
@@ -13,6 +17,29 @@ module Timescaledb
       timescale_continuous_aggregates(stream) # Define these before any Scenic views that might use them
     end
 
+    # Ignores Timescale related schemas when dumping the schema
+    IGNORE_SCHEMAS = %w[
+      _timescaledb_cache
+      _timescaledb_config
+      _timescaledb_catalog
+      _timescaledb_debug
+      _timescaledb_functions
+      _timescaledb_internal
+      timescaledb_experimental
+      timescaledb_information
+      toolkit_experimental
+    ]
+
+    def schemas(stream)
+      schema_names = @connection.schema_names - ["public", *IGNORE_SCHEMAS]
+      if schema_names.any?
+        schema_names.sort.each do |name|
+          stream.puts "  create_schema #{name.inspect}"
+        end
+        stream.puts
+      end
+    end
+
     def timescale_hypertables(stream)
       sorted_hypertables.each do |hypertable|
         timescale_hypertable(hypertable, stream)
@@ -112,19 +139,20 @@ module Timescaledb
     def timescale_continuous_aggregates(stream)
       return unless Timescaledb::ContinuousAggregates.table_exists?
 
-      Timescaledb::ContinuousAggregates.all.each do |aggregate|
-        opts = if (refresh_policy = aggregate.jobs.refresh_continuous_aggregate.first)
-                 interval = timescale_interval(refresh_policy.schedule_interval)
-                 end_offset = timescale_interval(refresh_policy.config["end_offset"])
-                 start_offset = timescale_interval(refresh_policy.config["start_offset"])
-                 %Q[, refresh_policies: { start_offset: "#{start_offset}", end_offset: "#{end_offset}", schedule_interval: "#{interval}"}]
-               else
-                 ""
-               end
+      Timescaledb::ContinuousAggregates.all.find_each do |aggregate|
+        refresh_policies_opts = if (refresh_policy = aggregate.jobs.refresh_continuous_aggregate.first)
+          interval = timescale_interval(refresh_policy.schedule_interval)
+          end_offset = timescale_interval(refresh_policy.config["end_offset"])
+          start_offset = timescale_interval(refresh_policy.config["start_offset"])
+          %(refresh_policies: { start_offset: "#{start_offset}", end_offset: "#{end_offset}", schedule_interval: "#{interval}"})
+        else
+          ""
+        end
 
+        with_clause_opts = "materialized_only: #{aggregate[:materialized_only]}, finalized: #{aggregate[:finalized]}"
         stream.puts <<~AGG.indent(2)
-          create_continuous_aggregate("#{aggregate.view_name}", <<-SQL#{opts})
-            #{aggregate.view_definition.strip.gsub(/;$/, "")}
+          create_continuous_aggregate("#{aggregate.view_name}", <<-SQL, #{refresh_policies_opts}, #{with_clause_opts})
+            #{aggregate.view_definition.strip.gsub(/;$/, '')}
           SQL
         AGG
         stream.puts

data/lib/timescaledb/stats/chunks.rb

@@ -0,0 +1,41 @@
+module Timescaledb
+  class Stats
+    class Chunks
+      # @param [Array<String>] hypertables The list of hypertable names.
+      # @param [Timescaledb:Connection] connection The PG connection.
+      def initialize(hypertables = [], connection = Timescaledb.connection)
+        @connection = connection
+        @hypertables = hypertables
+      end
+
+      delegate :query_count, to: :@connection
+
+      # @return [Hash] The chunks stats
+      def to_h
+        { total: total, compressed: compressed, uncompressed: uncompressed }
+      end
+
+      private
+
+      def total
+        query_count(base_query, [@hypertables])
+      end
+
+      def compressed
+        compressed_query = [base_query, 'is_compressed'].join(' AND ')
+
+        query_count(compressed_query, [@hypertables])
+      end
+
+      def uncompressed
+        uncompressed_query = [base_query, 'NOT is_compressed'].join(' AND ')
+
+        query_count(uncompressed_query, [@hypertables])
+      end
+
+      def base_query
+        "SELECT COUNT(1) FROM timescaledb_information.chunks WHERE hypertable_name IN ($1)"
+      end
+    end
+  end
+end

data/lib/timescaledb/stats/continuous_aggregates.rb

@@ -0,0 +1,24 @@
+
+module Timescaledb
+  class Stats
+    class ContinuousAggregates
+      # @param [Timescaledb:Connection] connection The PG connection.
+      def initialize(connection = Timescaledb.connection)
+        @connection = connection
+      end
+
+      delegate :query_count, to: :@connection
+
+      # @return [Hash] The continuous_aggregates stats
+      def to_h
+        { total: total }
+      end
+
+      private
+
+      def total
+        query_count('SELECT COUNT(1) FROM timescaledb_information.continuous_aggregates')
+      end
+    end
+  end
+end

data/lib/timescaledb/stats/hypertables.rb

@@ -0,0 +1,102 @@
+require_relative './chunks'
+
+module Timescaledb
+  class Stats
+    class Hypertables
+      # @param [Timescaledb:Connection] connection The PG connection.
+      # @param [Array<String>] hypertables The list of hypertable names.
+      def initialize(hypertables = [], connection = Timescaledb.connection)
+        @connection = connection
+        @hypertables = hypertables.map(&method('hypertable_name_with_schema'))
+      end
+
+      delegate :query, :query_first, :query_count, to: :@connection
+
+      # @return [Hash] The hypertables stats
+      def to_h
+        {
+          count: @hypertables.count,
+          uncompressed_count: uncompressed_count,
+          approximate_row_count: approximate_row_count,
+          chunks: Timescaledb::Stats::Chunks.new(@hypertables).to_h,
+          size: size
+        }
+      end
+
+      private
+
+      def uncompressed_count
+        @hypertables.count do |hypertable|
+          query("SELECT * from hypertable_compression_stats('#{hypertable}')").empty?
+        end
+      end
+
+      def approximate_row_count
+        @hypertables.each_with_object(Hash.new) do |hypertable, summary|
+          row_count = query_first("SELECT * FROM approximate_row_count('#{hypertable}')").approximate_row_count.to_i
+
+          summary[hypertable] = row_count
+        end
+      end
+
+      def size
+        sum = -> (method_name) { (@hypertables.map(&method(method_name)).inject(:+) || 0) }
+
+        {
+          uncompressed: humanize_bytes(sum[:before_total_bytes]),
+          compressed: humanize_bytes(sum[:after_total_bytes])
+        }
+      end
+
+      def before_total_bytes(hypertable)
+        (compression_stats[hypertable]&.before_compression_total_bytes || detailed_size[hypertable]).to_i
+      end
+
+      def after_total_bytes(hypertable)
+        (compression_stats[hypertable]&.after_compression_total_bytes || 0).to_i
+      end
+
+      def compression_stats
+        @compression_stats ||=
+          @hypertables.each_with_object(Hash.new) do |hypertable, stats|
+            stats[hypertable] = query_first(compression_stats_query, [hypertable])
+            stats
+          end
+      end
+
+      def compression_stats_query
+        'SELECT * FROM hypertable_compression_stats($1)'
+      end
+
+      def detailed_size
+        @detailed_size ||=
+          @hypertables.each_with_object(Hash.new) do |hypertable, size|
+            size[hypertable] = query_first(detailed_size_query, [hypertable]).total_bytes
+            size
+          end
+      end
+
+      def detailed_size_query
+        'SELECT * FROM hypertable_detailed_size($1)'
+      end
+
+      def hypertable_name_with_schema(hypertable)
+        [hypertable.hypertable_schema, hypertable.hypertable_name].compact.join('.')
+      end
+
+      def humanize_bytes(bytes)
+        units = %w(B KiB MiB GiB TiB PiB EiB)
+
+        return '0 B' if bytes == 0
+
+        exp = (Math.log2(bytes) / 10).floor
+        max_exp = units.size - 1
+        exp = max_exp if exp > max_exp
+
+        value = (bytes.to_f / (1 << (exp * 10))).round(1)
+
+        "#{value} #{units[exp]}"
+      end
+    end
+  end
+end

data/lib/timescaledb/stats/job_stats.rb

@@ -0,0 +1,29 @@
+
+module Timescaledb
+  class Stats
+    class JobStats
+      # @param [Timescaledb:Connection] connection The PG connection.
+      def initialize(connection = Timescaledb.connection)
+        @connection = connection
+      end
+
+      delegate :query_first, to: :@connection
+
+      # @return [Hash] The job_stats stats
+      def to_h
+        query_first(job_stats_query).to_h.transform_values(&:to_i)
+      end
+
+      private
+
+      def job_stats_query
+        <<-SQL
+          SELECT SUM(total_successes)::INT AS success,
+                 SUM(total_runs)::INT AS runs,
+                 SUM(total_failures)::INT AS failures
+          FROM timescaledb_information.job_stats
+        SQL
+      end
+    end
+  end
+end

data/lib/timescaledb/stats.rb

@@ -0,0 +1,22 @@
+require_relative './stats/continuous_aggregates'
+require_relative './stats/hypertables'
+require_relative './stats/job_stats'
+
+module Timescaledb
+  class Stats
+    # @param [Array<OpenStruct>] hypertables The list of hypertables.
+    # @param [Timescaledb:Connection] connection The PG connection.
+    def initialize(hypertables = [], connection = Timescaledb.connection)
+      @hypertables = hypertables
+      @connection = connection
+    end
+
+    def to_h
+      {
+        hypertables: Hypertables.new(@hypertables).to_h,
+        continuous_aggregates: ContinuousAggregates.new.to_h,
+        jobs_stats: JobStats.new.to_h
+      }
+    end
+  end
+end

data/lib/timescaledb/toolkit/time_vector.rb

@@ -42,7 +42,7 @@ module Timescaledb
               SELECT #{"x.#{segment_by}," if segment_by}
                 (lttb( x.#{time_column}, x.#{value_column}, #{threshold}) -> unnest()).*
               FROM x
-              #{"GROUP BY device_id" if segment_by}
+              #{"GROUP BY #{segment_by}" if segment_by}
             SQL
             downsampled = unscoped
               .select(*segment_by, "time as #{time_column}, value as #{value_column}")
@@ -59,35 +59,39 @@ module Timescaledb
           end
 
 
-          scope :_ohlc, -> (timeframe: '1h',
+          scope :_candlestick, -> (timeframe: '1h',
                            segment_by: segment_by_column,
                            time: time_column,
+                           volume: 'volume',
                            value: value_column) do
 
-             select( "time_bucket('#{timeframe}', #{time}) as #{time}",
-                           *segment_by,
-                           "toolkit_experimental.ohlc(#{time}, #{value})")
+             select(  %|time_bucket('#{timeframe}', "#{time}")|,
+                 *segment_by,
+                "candlestick_agg(#{time}, #{value}, #{volume}) as candlestick")
               .order(1)
               .group(*(segment_by ? [1,2] : 1))
           end
 
-          scope :ohlc, -> (timeframe: '1h',
+          scope :candlestick, -> (timeframe: '1h',
                            segment_by: segment_by_column,
                            time: time_column,
+                           volume:  'volume',
                            value: value_column) do
 
-            raw = _ohlc(timeframe: timeframe, segment_by: segment_by, time: time, value: value)
+            raw = _candlestick(timeframe: timeframe, segment_by: segment_by, time: time, value: value,  volume: volume)
             unscoped
-              .from("(#{raw.to_sql}) AS ohlc")
-              .select(*segment_by, time,
-               "toolkit_experimental.open(ohlc),
-                toolkit_experimental.high(ohlc),
-                toolkit_experimental.low(ohlc),
-                toolkit_experimental.close(ohlc),
-                toolkit_experimental.open_time(ohlc),
-                toolkit_experimental.high_time(ohlc),
-                toolkit_experimental.low_time(ohlc),
-                toolkit_experimental.close_time(ohlc)")
+              .from("(#{raw.to_sql}) AS candlestick")
+              .select("time_bucket",*segment_by,
+               "open(candlestick),
+                high(candlestick),
+                low(candlestick),
+                close(candlestick),
+                open_time(candlestick),
+                high_time(candlestick),
+                low_time(candlestick),
+                close_time(candlestick),
+                volume(candlestick),
+                vwap(candlestick)")
           end
         end
       end

data/lib/timescaledb/version.rb

@@ -1,3 +1,3 @@
 module Timescaledb
-  VERSION = '0.2.6'
+  VERSION = '0.2.8'
 end

data/lib/timescaledb.rb

@@ -3,15 +3,18 @@ require 'active_record'
 require_relative 'timescaledb/application_record'
 require_relative 'timescaledb/acts_as_hypertable'
 require_relative 'timescaledb/acts_as_hypertable/core'
+require_relative 'timescaledb/connection'
 require_relative 'timescaledb/toolkit'
 require_relative 'timescaledb/chunk'
 require_relative 'timescaledb/compression_settings'
+require_relative 'timescaledb/connection_handling'
 require_relative 'timescaledb/continuous_aggregates'
 require_relative 'timescaledb/dimensions'
 require_relative 'timescaledb/hypertable'
 require_relative 'timescaledb/job'
 require_relative 'timescaledb/job_stats'
 require_relative 'timescaledb/schema_dumper'
+require_relative 'timescaledb/stats'
 require_relative 'timescaledb/stats_report'
 require_relative 'timescaledb/migration_helpers'
 require_relative 'timescaledb/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timescaledb
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.8
 platform: ruby
 authors:
 - Jônatas Davi Paganini
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-03 00:00:00.000000000 Z
+date: 2024-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -144,99 +144,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/ci.yml"
-- ".gitignore"
-- ".rspec"
-- ".ruby-version"
-- ".tool-versions"
-- ".travis.yml"
-- CODE_OF_CONDUCT.md
-- Fastfile
-- Gemfile
-- Gemfile.lock
-- Gemfile.scenic
-- Gemfile.scenic.lock
 - LICENSE.txt
-- README.md
-- Rakefile
-- bin/console
-- bin/setup
 - bin/tsdb
-- docs/command_line.md
-- docs/img/lttb_example.png
-- docs/img/lttb_sql_vs_ruby.gif
-- docs/img/lttb_zoom.gif
-- docs/index.md
-- docs/migrations.md
-- docs/models.md
-- docs/toolkit.md
-- docs/toolkit_lttb_tutorial.md
-- docs/toolkit_lttb_zoom.md
-- docs/toolkit_ohlc.md
-- docs/videos.md
-- examples/all_in_one/all_in_one.rb
-- examples/all_in_one/benchmark_comparison.rb
-- examples/all_in_one/caggs.rb
-- examples/all_in_one/query_data.rb
-- examples/ranking/.gitattributes
-- examples/ranking/.gitignore
-- examples/ranking/.ruby-version
-- examples/ranking/Gemfile
-- examples/ranking/Gemfile.lock
-- examples/ranking/README.md
-- examples/ranking/Rakefile
-- examples/ranking/app/controllers/application_controller.rb
-- examples/ranking/app/controllers/concerns/.keep
-- examples/ranking/app/jobs/application_job.rb
-- examples/ranking/app/models/application_record.rb
-- examples/ranking/app/models/concerns/.keep
-- examples/ranking/app/models/game.rb
-- examples/ranking/app/models/play.rb
-- examples/ranking/bin/bundle
-- examples/ranking/bin/rails
-- examples/ranking/bin/rake
-- examples/ranking/bin/setup
-- examples/ranking/config.ru
-- examples/ranking/config/application.rb
-- examples/ranking/config/boot.rb
-- examples/ranking/config/credentials.yml.enc
-- examples/ranking/config/database.yml
-- examples/ranking/config/environment.rb
-- examples/ranking/config/environments/development.rb
-- examples/ranking/config/environments/production.rb
-- examples/ranking/config/environments/test.rb
-- examples/ranking/config/initializers/cors.rb
-- examples/ranking/config/initializers/filter_parameter_logging.rb
-- examples/ranking/config/initializers/inflections.rb
-- examples/ranking/config/initializers/timescale.rb
-- examples/ranking/config/locales/en.yml
-- examples/ranking/config/puma.rb
-- examples/ranking/config/routes.rb
-- examples/ranking/config/storage.yml
-- examples/ranking/db/migrate/20220209120747_create_games.rb
-- examples/ranking/db/migrate/20220209120910_create_plays.rb
-- examples/ranking/db/migrate/20220209143347_create_score_per_hours.rb
-- examples/ranking/db/schema.rb
-- examples/ranking/db/seeds.rb
-- examples/ranking/db/views/score_per_hours_v01.sql
-- examples/ranking/lib/tasks/.keep
-- examples/ranking/log/.keep
-- examples/ranking/public/robots.txt
-- examples/ranking/storage/.keep
-- examples/ranking/tmp/.keep
-- examples/ranking/tmp/pids/.keep
-- examples/ranking/tmp/storage/.keep
-- examples/ranking/vendor/.keep
-- examples/toolkit-demo/compare_volatility.rb
-- examples/toolkit-demo/lttb-zoom/README.md
-- examples/toolkit-demo/lttb-zoom/lttb_zoomable.rb
-- examples/toolkit-demo/lttb-zoom/views/index.erb
-- examples/toolkit-demo/lttb/README.md
-- examples/toolkit-demo/lttb/lttb.rb
-- examples/toolkit-demo/lttb/lttb_sinatra.rb
-- examples/toolkit-demo/lttb/lttb_test.rb
-- examples/toolkit-demo/lttb/views/index.erb
-- examples/toolkit-demo/ohlc.rb
 - lib/timescaledb.rb
 - lib/timescaledb/acts_as_hypertable.rb
 - lib/timescaledb/acts_as_hypertable/core.rb
@@ -244,7 +153,15 @@ files:
 - lib/timescaledb/application_record.rb
 - lib/timescaledb/chunk.rb
 - lib/timescaledb/compression_settings.rb
+- lib/timescaledb/connection.rb
+- lib/timescaledb/connection_handling.rb
 - lib/timescaledb/continuous_aggregates.rb
+- lib/timescaledb/database.rb
+- lib/timescaledb/database/chunk_statements.rb
+- lib/timescaledb/database/hypertable_statements.rb
+- lib/timescaledb/database/quoting.rb
+- lib/timescaledb/database/schema_statements.rb
+- lib/timescaledb/database/types.rb
 - lib/timescaledb/dimensions.rb
 - lib/timescaledb/hypertable.rb
 - lib/timescaledb/job.rb
@@ -253,13 +170,16 @@ files:
 - lib/timescaledb/scenic/adapter.rb
 - lib/timescaledb/scenic/extension.rb
 - lib/timescaledb/schema_dumper.rb
+- lib/timescaledb/stats.rb
+- lib/timescaledb/stats/chunks.rb
+- lib/timescaledb/stats/continuous_aggregates.rb
+- lib/timescaledb/stats/hypertables.rb
+- lib/timescaledb/stats/job_stats.rb
 - lib/timescaledb/stats_report.rb
 - lib/timescaledb/toolkit.rb
 - lib/timescaledb/toolkit/helpers.rb
 - lib/timescaledb/toolkit/time_vector.rb
 - lib/timescaledb/version.rb
-- mkdocs.yml
-- timescaledb.gemspec
 homepage: https://github.com/jonatas/timescaledb
 licenses:
 - MIT

data/.github/workflows/ci.yml

@@ -1,72 +0,0 @@
-name: CI
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-#  schedule:
-#    - cron: '42 5 * * *'
-
-jobs:
-  test-in-container:
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.1.2' ]
-        database:
-          - 'pg14.6-ts2.9.0-patroni-static-primary-latest'
-          - 'pg13.9-ts2.9-latest'
-
-    services:
-      database:
-        image: timescale/timescaledb-ha:${{matrix.database}}
-        env:
-          POSTGRES_USER: username
-          POSTGRES_PASSWORD: secret
-          POSTGRES_DB: testdb
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-
-    runs-on: ubuntu-latest
-    name: OS Ruby ${{matrix.ruby}} database ${{matrix.database}}
-    container: ruby:${{matrix.ruby}}
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Show Ruby Version
-      run: |
-        ruby -v
-
-    - name: Install psql
-      run: |
-        apt-get update
-        apt-get install -y postgresql-client
-
-    - name: Show PostgreSQL version and time
-      env:
-        PGPASSWORD: secret
-      run: |
-        echo "SELECT version()" | psql -h database -U username testdb
-        echo "SELECT CURRENT_TIME" | psql -h database -U username testdb
-
-    - name: Setup
-      run: |
-        ./bin/setup
-
-    - name: run tsdb
-      run: ./bin/tsdb postgres://username:secret@database:5432/testdb --stats
-
-    - name: Test setup
-      run: |
-        echo PG_URI_TEST="postgres://username:secret@database:5432/testdb" > .env
-        cat .env
-        bundle exec rake test:setup
-
-    - name: Test
-      run: bundle exec rake
-

data/.gitignore

@@ -1,12 +0,0 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-
-# rspec failure tracking
-.rspec_status
-.env

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper

data/.ruby-version

@@ -1 +0,0 @@
-3.1.2

data/.tool-versions

@@ -1 +0,0 @@
-ruby 2.7.1

data/.travis.yml

@@ -1,9 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 3.1.2
-before_install: gem install bundler
-gemfile:
-  - Gemfile
-  - Gemfile.scenic