timescaledb 0.2.0 → 0.2.2

data/examples/all_in_one/query_data.rb

@@ -0,0 +1,78 @@
+require 'bundler/inline' #require only what you need
+
+gemfile(true) do 
+  gem 'timescaledb', path:  '../..'
+  gem 'pry'
+  gem 'faker'
+end
+
+require 'pp'
+# ruby all_in_one.rb postgres://user:pass@host:port/db_name
+ActiveRecord::Base.establish_connection( ARGV.last)
+
+# Simple example
+class Event < ActiveRecord::Base
+  self.primary_key = nil
+  acts_as_hypertable
+
+# If you want to override the automatic assingment of the `created_at ` time series column
+  def self.timestamp_attributes_for_create_in_model
+    []
+  end
+  def self.timestamp_attributes_for_update_in_model
+    []
+  end
+end
+
+# Setup Hypertable as in a migration
+ActiveRecord::Base.connection.instance_exec do
+  ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+  drop_table(Event.table_name) if Event.table_exists?
+
+  hypertable_options = {
+    time_column: 'created_at',
+    chunk_time_interval: '7 day',
+    compress_segmentby: 'identifier',
+    compression_interval: '7 days'
+  }
+
+  create_table(:events, id: false, hypertable: hypertable_options) do |t|
+    t.string :identifier, null: false
+    t.jsonb :payload
+    t.timestamps
+  end
+end
+
+def generate_fake_data(total: 100_000)
+  time = 1.month.ago
+  total.times.flat_map do
+    identifier = %w[sign_up login click scroll logout view]
+    time = time + rand(60).seconds
+    {
+      created_at: time,
+      updated_at: time,
+      identifier: identifier.sample,
+      payload: {
+        "name" => Faker::Name.name,
+        "email" => Faker::Internet.email
+      }
+    }
+  end
+end
+
+
+batch = generate_fake_data total: 10_000
+ActiveRecord::Base.logger = nil
+Event.insert_all(batch, returning: false)
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+pp Event.previous_month.count
+pp Event.previous_week.count
+pp Event.previous_month.group('identifier').count
+pp Event.previous_week.group('identifier').count
+
+pp Event
+  .previous_month
+  .select("time_bucket('1 day', created_at) as time, identifier, count(*)")
+  .group("1,2").map(&:attributes)

data/examples/ranking/config/initializers/timescale.rb

@@ -1,3 +1,2 @@
-require 'timescale'
+require 'timescaledb'
 require 'scenic'
-

data/examples/toolkit-demo/compare_volatility.rb

@@ -0,0 +1,64 @@
+require 'bundler/setup'
+require 'timescaledb'
+
+
+# Compare volatility processing in Ruby vs SQL.
+class Measurement < ActiveRecord::Base
+  acts_as_hypertable time_column: "ts"
+  acts_as_time_vector segment_by: "device_id", value_column: "val"
+
+  scope :volatility_sql, -> do
+    select("device_id, timevector(#{time_column}, #{value_column}) -> sort() -> delta() -> abs() -> sum() as volatility")
+     .group("device_id")
+  end
+
+  scope :volatility_ruby, -> {
+    volatility = Hash.new(0)
+    previous = Hash.new
+    find_all do |measurement|
+      device_id = measurement.device_id
+      if previous[device_id]
+        delta = (measurement.val - previous[device_id]).abs
+        volatility[device_id] += delta
+      end
+      previous[device_id] = measurement.val
+    end
+    volatility
+  }
+end
+
+ActiveRecord::Base.establish_connection ENV["PG_URI"]
+ActiveRecord::Base.connection.add_toolkit_to_search_path!
+
+
+ActiveRecord::Base.connection.instance_exec do
+  ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+  unless Measurement.table_exists?
+    hypertable_options = {
+      time_column: 'ts',
+      chunk_time_interval: '1 day',
+    }
+    create_table :measurements, hypertable: hypertable_options, id: false do |t|
+      t.integer :device_id
+      t.decimal :val
+      t.timestamp :ts
+    end
+  end
+end
+
+if Measurement.count.zero?
+  ActiveRecord::Base.connection.execute(<<~SQL)
+      INSERT INTO measurements (ts, device_id, val)
+        SELECT ts, device_id, random()*80
+      FROM generate_series(TIMESTAMP '2022-01-01 00:00:00',
+                            TIMESTAMP '2022-02-01 00:00:00',
+                            INTERVAL '5 minutes') AS g1(ts),
+        generate_series(0, 5) AS g2(device_id);
+     SQL
+end
+
+Benchmark.bm do |x|
+  x.report("ruby")  { Measurement.volatility_ruby }
+  x.report("sql") { Measurement.volatility_sql.map(&:attributes)  }
+end

data/examples/toolkit-demo/lttb/README.md

@@ -0,0 +1,15 @@
+# LTTB examples
+
+This folder contains a few ideas to explore and learn more about the lttb algorithm.
+
+There is a [./lttb.rb](./lttb.rb) file that is the Ruby implementation of lttb
+and also contains the related [./lttb_test.rb](./lttb_test.rb) file that
+verifies the same example from the Timescale Toolkit [implementation](https://github.com/timescale/timescaledb-toolkit/blob/6ee2ea1e8ff64bab10b90bdf4cd4b0f7ed763934/extension/src/lttb.rs#L512-L530).
+
+The [./lttb_sinatra.rb](./lttb_sinatra.rb) is a small webserver that compares
+the SQL vs Ruby implementation. It also uses the [./views](./views) folder which
+contains the view rendering part.
+
+You can learn more by reading the [LTTB tutorial](https://jonatas.github.io/timescaledb/toolkit_lttb_tutorial/).
+
+

data/examples/toolkit-demo/lttb/lttb.rb

@@ -0,0 +1,92 @@
+module Triangle
+  module_function
+  def area(a, b, c)
+    (ax, ay),(bx,by),(cx,cy) = a,b,c
+    (
+      (ax - cx).to_f * (by - ay) -
+      (ax - bx).to_f * (cy - ay)
+    ).abs * 0.5
+  end
+end
+class Lttb
+  class << self
+    def avg(array)
+      array.sum.to_f / array.size
+    end
+
+    def downsample(data, threshold)
+      new(data, threshold).downsample
+    end
+  end
+
+  attr_reader :data, :threshold
+  def initialize(data, threshold)
+    fail 'data is not an array' unless data.is_a? Array
+    fail "threshold should be >= 2. It's #{threshold}." if threshold < 2
+    @data = data
+    @threshold = threshold
+  end
+
+  def downsample
+    case @data.first.first
+    when Time, DateTime, Date
+      transformed_dates = true
+      dates_to_numbers()
+    end
+    process.tap do |downsampled|
+      numbers_to_dates(downsampled) if transformed_dates
+    end
+  end
+  private
+
+  def process
+    return data if threshold >= data.size || threshold == 0
+
+    sampled = [data.first, data.last] # Keep first and last point. append in the middle.
+    point_index = 0
+
+    (threshold - 2).times do |i|
+      step = [((i+1.0) * bucket_size).to_i, data.size].min
+      next_point = (i * bucket_size).to_i  + 1
+
+      break if next_point > data.size - 2
+
+      points = data[step, slice]
+      avg_x = Lttb.avg(points.map(&:first)).to_i
+      avg_y = Lttb.avg(points.map(&:last))
+
+      max_area = -1.0
+
+      (next_point...(step + 1)).each do |idx|
+        area = Triangle.area(data[point_index], data[idx], [avg_x, avg_y])
+
+        if area > max_area
+          max_area = area
+          next_point = idx
+        end
+      end
+
+      sampled.insert(-2, data[next_point])
+      point_index = next_point
+    end
+
+    sampled
+  end
+
+  def bucket_size
+    @bucket_size ||= ((data.size - 2.0) / (threshold - 2.0))
+  end
+
+  def slice
+    @slice ||= bucket_size.to_i
+  end
+
+  def dates_to_numbers
+    @start_date = data[0][0].dup
+    data.each{|d| d[0] = d[0] - @start_date }
+  end
+
+  def numbers_to_dates(downsampled)
+    downsampled.each{|d| d[0] = @start_date + d[0]}
+  end
+end

data/examples/toolkit-demo/lttb/lttb_sinatra.rb

@@ -0,0 +1,139 @@
+# ruby lttb.rb postgres://user:pass@host:port/db_name
+require 'bundler/inline' #require only what you need
+
+gemfile(true) do
+  gem 'timescaledb', path:  '../../..'
+  gem 'pry'
+  gem 'sinatra', require: false
+  gem 'sinatra-reloader', require: false
+  gem 'sinatra-cross_origin', require: false
+  gem 'chartkick'
+end
+
+require 'timescaledb/toolkit'
+require 'sinatra'
+require 'sinatra/json'
+require 'sinatra/cross_origin'
+require 'chartkick'
+require_relative 'lttb'
+
+PG_URI = ARGV.last
+
+VALID_SIZES = %i[small med big]
+def download_weather_dataset size: :small
+  unless VALID_SIZES.include?(size)
+    fail "Invalid size: #{size}. Valids are #{VALID_SIZES}"
+  end
+  url = "https://timescaledata.blob.core.windows.net/datasets/weather_#{size}.tar.gz"
+  puts "fetching #{size} weather dataset..."
+  system "wget \"#{url}\""
+  puts "done!"
+end
+
+def setup size: :small
+  file = "weather_#{size}.tar.gz"
+  download_weather_dataset(size: size) unless File.exists? file
+  puts "extracting #{file}"
+  system "tar -xvzf #{file} "
+  puts "creating data structures"
+  system "psql #{PG_URI} < weather.sql"
+  system %|psql #{PG_URI} -c "\\COPY locations FROM weather_#{size}_locations.csv CSV"|
+  system %|psql #{PG_URI} -c "\\COPY conditions FROM weather_#{size}_conditions.csv CSV"|
+end
+
+ActiveRecord::Base.establish_connection(PG_URI)
+class Location < ActiveRecord::Base
+  self.primary_key = "device_id"
+
+  has_many :conditions, foreign_key: "device_id"
+end
+
+class Condition < ActiveRecord::Base
+  acts_as_hypertable time_column: "time"
+  acts_as_time_vector value_column: "temperature", segment_by: "device_id"
+
+  belongs_to :location, foreign_key: "device_id"
+end
+
+# Setup Hypertable as in a migration
+ActiveRecord::Base.connection.instance_exec do
+  ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+  unless Condition.table_exists?
+    setup size: :big
+  end
+end
+
+require 'sinatra/reloader'
+require 'sinatra/contrib'
+register Sinatra::Reloader
+register Sinatra::Contrib
+include Chartkick::Helper
+
+set :bind, '0.0.0.0'
+set :port, 9999
+
+def conditions
+  device_ids = (1..9).map{|i|"weather-pro-00000#{i}"}
+  Condition
+    .where(device_id: device_ids.first)
+    .order('time')
+end
+
+def threshold
+  params[:threshold]&.to_i || 50
+end
+
+configure do
+  enable :cross_origin
+end
+before do
+  response.headers['Access-Control-Allow-Origin'] = '*'
+end
+
+# routes...
+options "*" do
+  response.headers["Allow"] = "GET, PUT, POST, DELETE, OPTIONS"
+  response.headers["Access-Control-Allow-Headers"] = "Authorization, 
+        Content-Type, Accept, X-User-Email, X-Auth-Token"
+  response.headers["Access-Control-Allow-Origin"] = "*"
+  200
+end
+
+get '/' do
+  headers 'Access-Control-Allow-Origin' => 'https://cdn.jsdelivr.net/'
+
+  erb :index
+end
+
+get '/lttb_ruby' do
+  payload = conditions
+    .pluck(:device_id, :time, :temperature)
+    .group_by(&:first)
+    .map do |device_id, data|
+      data.each(&:shift)
+      {
+        name: device_id,
+        data: Lttb.downsample(data, threshold)
+      }
+  end
+  json payload
+end
+
+get "/lttb_sql" do
+  downsampled = conditions
+    .lttb(threshold: threshold)
+    .map do |device_id, data|
+      {
+        name: device_id,
+        data: data.sort_by(&:first)
+      }
+    end
+  json downsampled
+end
+
+
+get '/all_data' do
+  data = conditions.pluck(:time, :temperature)
+  json [ { name: "All data", data: data} ]
+end

data/examples/toolkit-demo/lttb/lttb_test.rb

@@ -0,0 +1,21 @@
+require_relative 'lttb'
+require 'pp'
+require 'date'
+
+data = [
+  ['2020-1-1', 10],
+  ['2020-1-2', 21],
+  ['2020-1-3', 19],
+  ['2020-1-4', 32],
+  ['2020-1-5', 12],
+  ['2020-1-6', 14],
+  ['2020-1-7', 18],
+  ['2020-1-8', 29],
+  ['2020-1-9', 23],
+  ['2020-1-10', 27],
+  ['2020-1-11', 14]]
+data.each do |e|
+  e[0] = Time.mktime(*e[0].split('-'))
+end
+
+pp Lttb.downsample(data, 5)

data/examples/toolkit-demo/lttb/views/index.erb

@@ -0,0 +1,27 @@
+<script src="https://cdn.jsdelivr.net/npm/jquery@3.6.1/dist/jquery.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/hammerjs@2.0.8hammerjs@2.0.8"></script>
+<script src="https://cdn.jsdelivr.net/npm/moment@2.29.4/moment.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/highcharts@10.2.1/highcharts.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-moment@1.0.0/dist/chartjs-adapter-moment.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/chartkick@4.2.0/dist/chartkick.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-zoom@1.2.1/dist/chartjs-plugin-zoom.min.js"></script>
+<h3>Downsampling <%= conditions.count %> records to
+  <select value="<%= threshold %>" onchange="location.href=`/?threshold=${this.value}`">
+    <option><%= threshold %></option>
+    <option value="50">50</option>
+    <option value="100">100</option>
+    <option value="500">500</option>
+    <option value="1000">1000</option>
+    <option value="5000">5000</option>
+  </select> points.
+</h3>
+
+<h3>SQL</h3>
+<%= line_chart("/lttb_sql?threshold=#{threshold}",
+              loading: "dowsampled data from SQL") %>
+<h3>Ruby</h3>
+<%= line_chart("/lttb_ruby?threshold=#{threshold}",
+               library: {chart: {zoomType: 'x'}},
+               points: true, loading: "downsampled data from Ruby") %>
+<!--%= line_chart("/all_data", loading: "Loading all data") %-->
+

data/examples/toolkit-demo/lttb-zoom/README.md

@@ -0,0 +1,13 @@
+# LTTB examples
+
+This folder contains an example to explore the a dynamic reloading of downsampled data.
+
+It keeps the same amount of data and refresh the data with a higher resolution
+as you keep zooming in. 
+There is a [./lttb_zoomable.rb](./lttb_zoomable.rb) file is a small webserver that compares
+the SQL vs Ruby implementation. It also uses the [./views](./views) folder which
+contains the view with the rendering and javascript part.
+
+You can learn more by reading the [LTTB Zoom tutorial](https://jonatas.github.io/timescaledb/toolkit_lttb_zoom/).
+
+

data/examples/toolkit-demo/lttb-zoom/lttb_zoomable.rb

@@ -0,0 +1,90 @@
+# ruby lttb_zoomable.rb postgres://user:pass@host:port/db_name
+require 'bundler/inline' #require only what you need
+
+gemfile(true) do
+  gem 'timescaledb', path:  '../../..'
+  gem 'pry'
+  gem 'sinatra', require: false
+  gem 'sinatra-reloader'
+  gem 'sinatra-cross_origin'
+end
+
+require 'timescaledb/toolkit'
+require 'sinatra'
+require 'sinatra/json'
+require 'sinatra/contrib'
+
+register Sinatra::Reloader
+register Sinatra::Contrib
+
+PG_URI = ARGV.last
+
+VALID_SIZES = %i[small med big]
+def download_weather_dataset size: :small
+  unless VALID_SIZES.include?(size)
+    fail "Invalid size: #{size}. Valids are #{VALID_SIZES}"
+  end
+  url = "https://timescaledata.blob.core.windows.net/datasets/weather_#{size}.tar.gz"
+  puts "fetching #{size} weather dataset..."
+  system "wget \"#{url}\""
+  puts "done!"
+end
+
+def setup size: :small
+  file = "weather_#{size}.tar.gz"
+  download_weather_dataset(size: size) unless File.exists? file
+  puts "extracting #{file}"
+  system "tar -xvzf #{file} "
+  puts "creating data structures"
+  system "psql #{PG_URI} < weather.sql"
+  system %|psql #{PG_URI} -c "\\COPY locations FROM weather_#{size}_locations.csv CSV"|
+  system %|psql #{PG_URI} -c "\\COPY conditions FROM weather_#{size}_conditions.csv CSV"|
+end
+
+ActiveRecord::Base.establish_connection(PG_URI)
+
+class Condition < ActiveRecord::Base
+  acts_as_hypertable time_column: "time"
+  acts_as_time_vector value_column: "temperature", segment_by: "device_id"
+end
+
+# Setup Hypertable as in a migration
+ActiveRecord::Base.connection.instance_exec do
+  ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+  if !Condition.table_exists?  || Condition.count.zero?
+
+    setup size: :big
+  end
+end
+
+
+def filter_by_request_params
+  filter= {device_id: "weather-pro-000001"}
+  if params[:filter] && params[:filter] != "null"
+    from, to = params[:filter].split(",").map(&Time.method(:parse))
+    filter[:time] = from..to
+  end
+  filter
+end
+
+def conditions
+  Condition.where(filter_by_request_params).order('time')
+end
+
+def threshold
+  params[:threshold]&.to_i || 50
+end
+
+configure do
+  enable :cross_origin
+end
+
+get '/' do
+  erb :index
+end
+
+get "/lttb_sql" do
+  downsampled = conditions.lttb(threshold: threshold, segment_by: nil)
+  json downsampled
+end

data/examples/toolkit-demo/lttb-zoom/views/index.erb

@@ -0,0 +1,33 @@
+<head>
+  <script src="https://cdn.jsdelivr.net/npm/jquery@3.6.1/dist/jquery.min.js"></script>
+  <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+</head>
+
+<h3>Downsampling <%= conditions.count %> records to
+  <select value="<%= threshold %>" onchange="location.href=`/?threshold=${this.value}`">
+    <option><%= threshold %></option>
+    <option value="50">50</option>
+    <option value="100">100</option>
+    <option value="500">500</option>
+    <option value="1000">1000</option>
+    <option value="5000">5000</option>
+  </select> points.
+</h3>
+<div id='container'></div>
+<script>
+  let chart = document.getElementById('container');
+  function fetch(filter) {
+    $.ajax({
+      url: `/lttb_sql?threshold=<%= threshold %>&filter=${filter}`,
+      success: function(result) {
+        let x = result.map((e) => e[0]);
+        let y = result.map((e) => parseFloat(e[1]));
+          Plotly.newPlot(chart, [{x, y,"mode": "markers", "type": "scatter"}]);
+        chart.on('plotly_relayout',
+          function(eventdata){
+            fetch([eventdata['xaxis.range[0]'],eventdata['xaxis.range[1]']]);
+          });
+      }});
+  }
+  fetch(null);
+</script>

data/lib/timescaledb/acts_as_time_vector.rb

@@ -0,0 +1,18 @@
+module Timescaledb
+  module ActsAsTimeVector
+    def acts_as_time_vector(options = {})
+      return if acts_as_time_vector?
+
+      include Timescaledb::Toolkit::TimeVector
+
+      class_attribute :time_vector_options, instance_writer: false
+      define_default_scopes
+      self.time_vector_options = options
+    end
+
+    def acts_as_time_vector?
+      included_modules.include?(Timescaledb::ActsAsTimeVector)
+    end
+  end
+end
+ActiveRecord::Base.extend Timescaledb::ActsAsTimeVector

data/lib/timescaledb/dimensions.rb

@@ -1,6 +1,7 @@
 module Timescaledb
   class Dimension < ActiveRecord::Base
     self.table_name = "timescaledb_information.dimensions"
+    attribute :time_interval, :interval
   end
   Dimensions = Dimension
 end

data/lib/timescaledb/hypertable.rb

@@ -10,7 +10,7 @@ module Timescaledb
       foreign_key: "hypertable_name",
       class_name: "Timescaledb::CompressionSetting"
 
-    has_one :dimensions,
+    has_many :dimensions,
       foreign_key: "hypertable_name",
       class_name: "Timescaledb::Dimension"
 
@@ -18,6 +18,10 @@ module Timescaledb
       foreign_key: "hypertable_name",
       class_name: "Timescaledb::ContinuousAggregate"
 
+    def main_dimension
+      dimensions.find_by dimension_number: 1
+    end
+
     def chunks_detailed_size
       struct_from "SELECT * from chunks_detailed_size('#{self.hypertable_name}')"
     end

data/lib/timescaledb/migration_helpers.rb

@@ -72,26 +72,45 @@ module Timescaledb
     #     GROUP BY bucket
     #   SQL
     #
-    def create_continuous_aggregate(name, query, **options)
+    def create_continuous_aggregate(table_name, query, **options)
       execute <<~SQL
-        CREATE MATERIALIZED VIEW #{name}
+        CREATE MATERIALIZED VIEW #{table_name}
         WITH (timescaledb.continuous) AS
         #{query.respond_to?(:to_sql) ? query.to_sql : query}
         WITH #{"NO" unless options[:with_data]} DATA;
       SQL
 
-      if (policy = options[:refresh_policies])
-        # TODO: assert valid keys
-        execute <<~SQL
-          SELECT add_continuous_aggregate_policy('#{name}',
-            start_offset => #{policy[:start_offset]},
-            end_offset => #{policy[:end_offset]},
-            schedule_interval => #{policy[:schedule_interval]});
-        SQL
-      end
+      create_continuous_aggregate_policy(table_name, options[:refresh_policies] || {})
+    end
+
+
+    #  Drop a new continuous aggregate.
+    #
+    #  It basically DROP MATERIALIZED VIEW for a given @name.
+    #
+    # @param name [String, Symbol] The name of the continuous aggregate view.
+    def drop_continuous_aggregates view_name
+      execute "DROP MATERIALIZED VIEW #{view_name}"
     end
+
     alias_method :create_continuous_aggregates, :create_continuous_aggregate
 
+    def create_continuous_aggregate_policy(table_name, **options)
+      return if options.empty?
+
+      # TODO: assert valid keys
+      execute <<~SQL
+        SELECT add_continuous_aggregate_policy('#{table_name}',
+          start_offset => #{options[:start_offset]},
+          end_offset => #{options[:end_offset]},
+          schedule_interval => #{options[:schedule_interval]});
+      SQL
+    end
+
+    def remove_continuous_aggregate_policy(table_name)
+      execute "SELECT remove_continuous_aggregate_policy('#{table_name}')"
+    end
+
     def create_retention_policy(table_name, interval:)
       execute "SELECT add_retention_policy('#{table_name}', INTERVAL '#{interval}')"
     end