timescaledb 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b622fffc9a920a95e1c0615df32fa13e6b60d21198ebd39a6c46d27a66e11df4
-  data.tar.gz: 9da077e24cb64120e1d235e05234299f9c2fb7cbbbd63e50a68a13bad3a23898
+  metadata.gz: c5f8ebd4460e965fbf9a35600630c05d476b1a4b674192cd925a3d5f948a64a1
+  data.tar.gz: d697ab124689a8c4f1ffc5b809a7cecd2ac07bbce84c7b9ca539dc5ae67068c9
 SHA512:
-  metadata.gz: b4c098c20e3a99f5c8798f5ed2e29c095b5982b55defc5612128a33bf840c01d7eb457df2a5cd83fc41243be9b48184a8bc9b585800d2d7daa28085793263246
-  data.tar.gz: b1420ac494f3a1ed6ebbd4ebd3701a9a3d33b122bbf3d6dd4b15c94159ce5299d24d0f08c68ef612b6758859b388811415593d4512fc39bfbff26876ec82efea
+  metadata.gz: 1070bf2f732137006d81790ac1c4b467f733edd3bb724e8773d3c9f6ecb3b5c1dc32c3da638f351589fa944a48c1b8f7a037da6a896beed1557e4fb34e2a8442
+  data.tar.gz: f5bc47e8c0022d079189e7ad68e2214da6760c4f420bca3c83950137dd7026985fb9239924c241b28f261546a860c744a32db154548bfc0f8871302d35109ff7

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+3.1.2

data/Fastfile

@@ -0,0 +1,17 @@
+
+# Use `fast .version_up` to rewrite the version file
+Fast.shortcut :version_up do
+  rewrite_file('(casgn nil VERSION (str _)', 'lib/timescaledb/version.rb') do |node|
+    target = node.children.last.loc.expression
+    pieces = target.source.split('.').map(&:to_i)
+    pieces.reverse.each_with_index do |fragment, i|
+      if fragment < 9
+        pieces[-(i + 1)] = fragment + 1
+        break
+      else
+        pieces[-(i + 1)] = 0
+      end
+    end
+    replace(target, "'#{pieces.join('.')}'")
+  end
+end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    timescaledb (0.2.1)
+    timescaledb (0.2.4)
       activerecord
       activesupport
       pg (~> 1.2)
@@ -33,7 +33,7 @@ GEM
       concurrent-ruby (~> 1.0)
     method_source (1.0.0)
     minitest (5.14.4)
-    pg (1.3.1)
+    pg (1.4.4)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)

data/Gemfile.scenic.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    timescaledb (0.1.5)
+    timescaledb (0.2.3)
       activerecord
       activesupport
       pg (~> 1.2)
@@ -58,7 +58,7 @@ GEM
       racc (~> 1.4)
     nokogiri (1.12.5-x86_64-darwin)
       racc (~> 1.4)
-    pg (1.3.0)
+    pg (1.4.4)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)

data/docs/index.md

@@ -40,6 +40,17 @@ The [all_in_one](https://github.com/jonatas/timescaledb/tree/master/examples/all
 
 The [ranking](https://github.com/jonatas/timescaledb/tree/master/examples/ranking) example shows how to configure a Rails app and navigate all the features available.
 
+
+## Toolkit  examples
+
+There are also examples in the [toolkit-demo](https://github.com/jonatas/timescaledb/tree/master/examples/toolkit-demo) folder that can help you to
+understand how to properly use the toolkit functions.
+
+* [ohlc](https://github.com/jonatas/timescaledb/tree/master/examples/toolkit-demo/ohlc.rb) is a funtion that groups data by Open, High, Low, Close and make histogram availables to group the data, very useful for financial analysis.
+* While building the [LTTB tutorial]( https://jonatas.github.io/timescaledb/toolkit_lttb_tutorial/) I created the [lttb](https://github.com/jonatas/timescaledb/tree/master/examples/toolkit-demo/lttb) is a simple charting using the Largest Triangle Three Buckets and there. A [zoomable](https://github.com/jonatas/timescaledb/tree/master/examples/toolkit-demo/lttb-zoom) version which allows to navigate in the data and zoom it keeping the same data resolution is also available.
+* A small example showing how to process [volatility](https://github.com/jonatas/timescaledb/blob/master/examples/toolkit-demo/compare_volatility.rb) is also good to get familiar with the pipeline functions. A benchmark implementing the same in Ruby is also available to check how it compares to the SQL implementation.
+
+
 ## Extra resources
 
 If you need extra help, please join the fantastic [timescale community](https://www.timescale.com/community)

data/docs/migrations.md

@@ -67,3 +67,10 @@ options = {
 create_continuous_aggregate('ohlc_1m', query, **options)
 ```
 
+If you need more details, please check this [blog post][1].
+
+If you're interested in candlesticks and need to get the OHLC values, take a look
+at the [toolkit ohlc](/toolkit_ohlc) function that do the same but through a
+function that can be reusing candlesticks from smaller timeframes.
+
+[1]: https://ideia.me/timescale-continuous-aggregates-with-ruby

data/docs/toolkit.md

@@ -93,7 +93,7 @@ Now, let's add the model `app/models/measurement.rb`:
 
 ```ruby
 class Measurement < ActiveRecord::Base
-  self.primary_key = 'device_id'
+  self.primary_key = nil
 
   acts_as_hypertable time_column: "ts"
 end
@@ -168,12 +168,15 @@ Measurement
 The final query for the example above looks like this:
 
 ```sql
-SELECT device_id, sum(abs_delta) as volatility
+SELECT device_id, SUM(abs_delta) AS volatility
 FROM (
   SELECT device_id,
-    abs(val - lag(val) OVER (PARTITION BY device_id ORDER BY ts)) as abs_delta
+    ABS(
+      val - LAG(val) OVER (
+        PARTITION BY device_id ORDER BY ts)
+      ) AS abs_delta
   FROM "measurements"
-) as calc_delta
+) AS calc_delta
 GROUP BY device_id
 ```
 
@@ -182,8 +185,14 @@ let's reproduce the same example using the toolkit pipelines:
 
 ```ruby
 Measurement
-  .select("device_id, timevector(ts, val) -> sort() -> delta() -> abs() -> sum() as volatility")
-  .group("device_id")
+  .select(<<-SQL).group("device_id")
+    device_id,
+    timevector(ts, val)
+      -> sort()
+      -> delta()
+      -> abs()
+      -> sum() as volatility
+    SQL
 ```
 
 As you can see, it's much easier to read and digest the example. Now, let's take
@@ -198,7 +207,7 @@ here to allow us to not repeat the parameters of the `timevector(ts, val)` call.
 
 ```ruby
 class Measurement < ActiveRecord::Base
-  self.primary_key = 'device_id'
+  self.primary_key = nil
 
   acts_as_hypertable time_column: "ts"
 
@@ -224,8 +233,14 @@ class Measurement < ActiveRecord::Base
     time_column: "ts"
 
   scope :volatility, -> do
-    select("device_id, timevector(#{time_column}, #{value_column}) -> sort() -> delta() -> abs() -> sum() as volatility")
-     .group("device_id")
+    select(<<-SQL).group("device_id")
+      device_id,
+      timevector(#{time_column}, #{value_column})
+        -> sort()
+        -> delta()
+        -> abs()
+        -> sum() as volatility
+    SQL
   end
 end
 ```
@@ -248,7 +263,12 @@ class Measurement < ActiveRecord::Base
 
   scope :volatility, -> (columns=segment_by_column) do
     _scope = select([*columns,
-        "timevector(#{time_column}, #{value_column}) -> sort() -> delta() -> abs() -> sum() as volatility"
+        "timevector(#{time_column},
+        #{value_column})
+           -> sort()
+           -> delta()
+           -> abs()
+           -> sum() as volatility"
     ].join(", "))
     _scope = _scope.group(columns) if columns
     _scope
@@ -361,7 +381,7 @@ Now, let's measure compare the time to process the volatility:
 ```ruby
 Benchmark.bm do |x|
   x.report("ruby")  { pp Measurement.volatility_by_device_id }
-  x.report("sql") { pp Measurement.volatility("device_id").map(&:attributes)  }
+  x.report("sql") { pp Measurement.volatility("device_id").map(&:attributes) }
 end
 #           user     system      total        real
 # ruby    0.612439   0.061890   0.674329 (  0.727590)
@@ -379,10 +399,103 @@ records over the wires. Now, moving to a remote host look the numbers:
     Now, using a remote connection between different regions,
     it looks even ~500 times slower than SQL.
 
-            user     system      total        real
-      ruby 0.716321   0.041640   0.757961 (  6.388881)
-      sql  0.001156   0.000177   0.001333 (  0.161270)
+                user     system      total        real
+        ruby 0.716321   0.041640   0.757961 (  6.388881)
+        sql  0.001156   0.000177   0.001333 (  0.161270)
 
+Let’s recap what’s time consuming here. The `find_all` is just not optimized to
+fetch the data and also consuming most of the time here. It’s also fetching
+the data and converting it to ActiveRecord model which has thousands of methods.
+
+It’s very comfortable but just need the attributes to make it.
+
+Let’s optimize it by plucking an array of values grouped by device.
+
+```ruby
+class Measurement < ActiveRecord::Base
+  # ...
+  scope :values_from_devices, -> {
+    ordered_values = select(:val, :device_id).order(:ts)
+    Hash[
+      from(ordered_values)
+      .group(:device_id)
+      .pluck("device_id, array_agg(val)")
+    ]
+  }
+end
+```
+
+Now, let's create a method for processing volatility.
+
+```ruby
+class Volatility
+  def self.process(values)
+    previous = nil
+    deltas = values.map do |value|
+      if previous
+        delta = (value - previous).abs
+        volatility = delta
+      end
+      previous = value
+      volatility
+    end
+    #deltas => [nil, 1, 1]
+    deltas.shift
+    volatility = deltas.sum
+  end
+  def self.process_values(map)
+    map.transform_values(&method(:process))
+  end
+end
+```
+
+Now, let's change the benchmark to expose the time for fetching and processing:
+
+
+```ruby
+volatilities = nil
+
+ActiveRecord::Base.logger = nil
+Benchmark.bm do |x|
+  x.report("ruby")  { Measurement.volatility_ruby }
+  x.report("sql") { Measurement.volatility_sql.map(&:attributes)  }
+  x.report("fetch") { volatilities =  Measurement.values_from_devices }
+  x.report("process") { Volatility.process_values(volatilities) }
+end
+```
+
+Checking the results:
+
+          user     system      total        real
+    ruby  0.683654   0.036558   0.720212 (  0.743942)
+    sql  0.000876   0.000096   0.000972 (  0.054234)
+    fetch  0.078045   0.003221   0.081266 (  0.116693)
+    process  0.067643   0.006473   0.074116 (  0.074122)
+
+Much better, now we can see only 200ms difference between real time which means ~36% more.
+
+
+If we try to break down a bit more of the SQL part, we can see that the 
+
+```sql
+EXPLAIN ANALYSE
+  SELECT device_id, array_agg(val)
+  FROM (
+    SELECT val, device_id
+    FROM measurements
+    ORDER BY ts ASC
+  ) subquery
+  GROUP BY device_id;
+```
+
+We can check the execution time and make it clear how much time is necessary
+just for the processing part, isolating network and the ActiveRecord layer.
+
+    │ Planning Time: 17.761 ms                                                                                                                                                                     │
+    │ Execution Time: 36.302 ms
+
+So, it means that from the **116ms** to fetch the data, only **54ms** was used from the DB
+and the remaining **62ms** was consumed by network + ORM.
 
 [1]: https://github.com/timescale/timescaledb-toolkit
 [2]: https://timescale.com

data/docs/toolkit_ohlc.md

@@ -0,0 +1,315 @@
+# OHLC / Candlesticks
+
+Candlesticks are a popular tool in technical analysis, used by traders to determine potential market movements.
+
+The toolkit also allows you to compute candlesticks with the [ohlc][1] function.
+
+Candlesticks are a type of price chart that displays the high, low, open, and close prices of a security for a specific period. They can be useful because they can provide information about market trends and reversals. For example, if you see that the stock has been trading in a range for a while, it may be worth considering buying or selling when the price moves outside of this range. Additionally, candlesticks can be used in conjunction with other technical indicators to make trading decisions.
+
+
+Let's start defining a table that stores the trades from financial market data
+and then we can calculate the candlesticks with the Timescaledb Toolkit.
+
+## Migration
+
+The `ticks` table is a hypertable that will be partitioning the data into one
+week intervl. Compressing them after a month to save storage.
+
+```ruby
+hypertable_options = {
+  time_column: 'time',
+  chunk_time_interval: '1 week',
+  compress_segmentby: 'symbol',
+  compress_orderby: 'time',
+  compression_interval: '1 month'
+}
+create_table :ticks, hypertable: hypertable_options, id: false do |t|
+  t.timestampt :time
+  t.string :symbol
+  t.decimal :price
+  t.integer :volume
+end
+```
+
+In the previous code block, we assume it goes inside a Rails migration or you
+can embed such code into a `ActiveRecord::Base.connection.instance_exec` block.
+
+## Defining the model
+
+As we don't need a primary key for the table, let's set it to nil. The
+`acts_as_hypertable` macro will give us several useful scopes that can be
+wrapping some of the TimescaleDB features.
+
+The `acts_as_time_vector` will allow us to set what are the default columns used
+to calculate the data.
+
+
+```ruby
+class Tick < ActiveRecord::Base
+  self.primary_key = nil
+  acts_as_hypertable time_column: :time
+  acts_as_time_vector value_column: price, segment_by: :symbol
+end
+```
+
+The candlestick will split the timeframe by the `time_column` and use the `price` as the default value to process the candlestick. It will also segment the candles by `symbol`.
+
+If you need to generate some data for your table, please check [this post][2].
+
+## The `ohlc` scope
+
+When the `acts_as_time_vector` method is used in the model, it will inject
+several scopes from the toolkit to easily have access to functions like the
+ohlc.
+
+The `ohlc` scope is available with a few parameters that inherits the
+configuration from the `acts_as_time_vector` declared previously.
+
+The simplest query is:
+
+```ruby
+Tick.ohlc(timeframe: '1m')
+```
+
+It will generate the following SQL:
+
+```sql
+ SELECT symbol,
+    "time",
+    toolkit_experimental.open(ohlc),
+    toolkit_experimental.high(ohlc),
+    toolkit_experimental.low(ohlc),
+    toolkit_experimental.close(ohlc),
+    toolkit_experimental.open_time(ohlc),
+    toolkit_experimental.high_time(ohlc),
+    toolkit_experimental.low_time(ohlc),
+    toolkit_experimental.close_time(ohlc)
+FROM (
+    SELECT time_bucket('1m', time) as time,
+      "ticks"."symbol",
+      toolkit_experimental.ohlc(time, price)
+    FROM "ticks" GROUP BY 1, 2 ORDER BY 1)
+AS ohlc
+```
+
+The timeframe argument can also be skipped and the default is `1 hour`.
+
+You can also combine other scopes to filter data before you get the data from the candlestick:
+
+```ruby
+Tick.yesterday
+  .where(symbol: "APPL")
+  .ohlc(timeframe: '1m')
+```
+
+The `yesterday` scope is automatically included because of the `acts_as_hypertable` macro. And it will be combining with other where clauses.
+
+## Continuous aggregates
+
+If you would like to continuous aggregate the candlesticks on a materialized
+view you can use continuous aggregates for it.
+
+The next examples shows how to create a continuous aggregates of 1 minute
+candlesticks:
+
+```ruby
+options = {
+  with_data: false,
+  refresh_policies: {
+    start_offset: "INTERVAL '1 month'",
+    end_offset: "INTERVAL '1 minute'",
+    schedule_interval: "INTERVAL '1 minute'"
+  }
+}
+create_continuous_aggregate('ohlc_1m', Tick.ohlc(timeframe: '1m'), **options)
+```
+
+
+Note that the `create_continuous_aggregate` calls the `to_sql` method in case
+the second parameter is not a string.
+
+## Rollup
+
+The rollup allows you to combine ohlc structures from smaller timeframes
+to bigger timeframes without needing to reprocess all the data.
+
+With this feature, you can group by the ohcl multiple times saving processing 
+from the server and make it easier to manage candlesticks from different time intervals.
+
+In the previous example, we used the `.ohlc` function that returns already the
+attributes from the different timeframes. In the SQL command it's calling the
+`open`, `high`, `low`, `close` functions that can access the values behind the
+ohlcsummary type.
+
+To merge the ohlc we need to rollup the `ohlcsummary` to a bigger timeframe and
+only access the values as a final resort to see them and access as attributes.
+
+Let's rebuild the structure:
+
+```ruby
+execute "CREATE VIEW ohlc_1h AS #{ Ohlc1m.rollup(timeframe: '1 hour').to_sql}"
+execute "CREATE VIEW ohlc_1d AS #{ Ohlc1h.rollup(timeframe: '1 day').to_sql}"
+```
+
+## Defining models for views
+
+Note that the previous code refers to `Ohlc1m` and `Ohlc1h` as two classes that
+are not defined yet. They will basically be ActiveRecord readonly models to
+allow to build scopes from it.
+
+Ohlc for one hour:
+```ruby
+class Ohlc1m < ActiveRecord::Base
+  self.table_name = 'ohlc_1m'
+  include Ohlc
+end
+```
+
+Ohlc for one day is pretty much the same:
+```ruby
+class Ohlc1h < ActiveRecord::Base
+  self.table_name = 'ohlc_1h'
+  include Ohlc
+end
+```
+
+We'll also have the `Ohlc` as a shared concern that can help you to reuse
+queries in different views.
+
+```ruby
+module Ohlc
+  extend ActiveSupport::Concern
+
+  included do
+    scope :rollup, -> (timeframe: '1h') do
+      select("symbol, time_bucket('#{timeframe}', time) as time,
+            toolkit_experimental.rollup(ohlc) as ohlc")
+      .group(1,2)
+    end
+
+    scope :attributes, -> do
+      select("symbol, time,
+        toolkit_experimental.open(ohlc),
+        toolkit_experimental.high(ohlc),
+        toolkit_experimental.low(ohlc),
+        toolkit_experimental.close(ohlc),
+        toolkit_experimental.open_time(ohlc),
+        toolkit_experimental.high_time(ohlc),
+        toolkit_experimental.low_time(ohlc),
+        toolkit_experimental.close_time(ohlc)")
+    end
+
+    # Following the attributes scope, we can define accessors in the
+    # model to populate from the previous scope to make it similar
+    # to a regular model structure.
+    attribute :time, :time
+    attribute :symbol, :string
+
+    %w[open high low close].each do |name|
+      attribute name, :decimal
+      attribute "#{name}_time", :time
+    end
+
+    def readonly?
+      true
+    end
+  end
+end
+```
+
+The `rollup` scope is the one that was used to redefine the data into big timeframes
+and the `attributes` allow to access the attributes from the [OpenHighLowClose][3]
+type.
+
+In this way, the views become just shortcuts and complex sql can also be done
+just nesting the model scope. For example, to rollup from a minute to a month,
+you can do:
+
+```ruby
+Ohlc1m.attributes.from(
+  Ohlc1m.rollup(timeframe: '1 month')
+)
+```
+
+Soon the continuous aggregates will [support nested aggregates][4] and you'll be
+abble to define the materialized views with steps like this:
+
+
+```ruby
+Ohlc1m.attributes.from(
+  Ohlc1m.rollup(timeframe: '1 month').from(
+    Ohlc1m.rollup(timeframe: '1 week').from(
+      Ohlc1m.rollup(timeframe: '1 day').from(
+        Ohlc1m.rollup(timeframe: '1 hour')
+      )
+    )
+  )
+)
+```
+
+For now composing the subqueries will probably be less efficient and unnecessary.
+But the foundation is already here to help you in future analysis. Just to make
+it clear, here is the SQL generated from the previous code:
+
+```sql
+SELECT symbol,
+    time,
+    toolkit_experimental.open(ohlc),
+    toolkit_experimental.high(ohlc),
+    toolkit_experimental.low(ohlc),
+    toolkit_experimental.close(ohlc),
+    toolkit_experimental.open_time(ohlc),
+    toolkit_experimental.high_time(ohlc),
+    toolkit_experimental.low_time(ohlc),
+    toolkit_experimental.close_time(ohlc)
+FROM (
+    SELECT symbol,
+        time_bucket('1 month', time) as time,
+        toolkit_experimental.rollup(ohlc) as ohlc
+    FROM (
+        SELECT symbol,
+            time_bucket('1 week', time) as time,
+            toolkit_experimental.rollup(ohlc) as ohlc
+        FROM (
+            SELECT symbol,
+                time_bucket('1 day', time) as time,
+                toolkit_experimental.rollup(ohlc) as ohlc
+            FROM (
+                SELECT symbol,
+                    time_bucket('1 hour', time) as time,
+                    toolkit_experimental.rollup(ohlc) as ohlc
+                FROM "ohlc_1m"
+                GROUP BY 1, 2
+            ) subquery
+            GROUP BY 1, 2
+        ) subquery
+        GROUP BY 1, 2
+    ) subquery
+    GROUP BY 1, 2
+) subquery
+```
+
+You can also define more scopes that will be useful depending on what are you
+working on. Example:
+
+```ruby
+scope :yesterday, -> { where("DATE(#{time_column}) = ?", Date.yesterday.in_time_zone.to_date) }
+```
+
+And then, just combine the scopes:
+
+```ruby
+Ohlc1m.yesterday.attributes
+```
+I hope you find this tutorial interesting and you can also check the
+`ohlc.rb` file in the [examples/toolkit-demo][5] folder.
+
+If you have any questions or concerns, feel free to reach me ([@jonatasdp][7]) in the [Timescale community][6] or tag timescaledb in your StackOverflow issue.
+
+[1]: https://docs.timescale.com/api/latest/hyperfunctions/financial-analysis/ohlc/
+[2]: https://ideia.me/timescale-continuous-aggregates-with-ruby
+[3]: https://github.com/timescale/timescaledb-toolkit/blob/cbbca7b2e69968e585c845924e7ed7aff1cea20a/extension/src/ohlc.rs#L20-L24
+[4]: https://github.com/timescale/timescaledb/pull/4668
+[5]: https://github.com/jonatas/timescaledb/tree/master/examples/toolkit-demo
+[6]: https://timescale.com/community
+[7]: https://twitter.com/jonatasdp

data/examples/toolkit-demo/compare_volatility.rb

@@ -1,6 +1,14 @@
-require 'bundler/setup'
-require 'timescaledb'
+# ruby compare_volatility.rb postgres://user:pass@host:port/db_name
+require 'bundler/inline' #require only what you need
 
+gemfile(true) do
+  gem 'timescaledb', path:  '../..'
+  gem 'pry'
+end
+
+# TODO: get the volatility using the window function with plain postgresql
+
+ActiveRecord::Base.establish_connection ARGV.last
 
 # Compare volatility processing in Ruby vs SQL.
 class Measurement < ActiveRecord::Base
@@ -25,9 +33,36 @@ class Measurement < ActiveRecord::Base
     end
     volatility
   }
+  scope :values_from_devices, -> {
+    ordered_values = select(:val, :device_id).order(:ts)
+    Hash[
+      from(ordered_values)
+      .group(:device_id)
+      .pluck("device_id, array_agg(val)")
+    ]
+  }
+end
+
+class Volatility
+  def self.process(values)
+    previous = nil
+    deltas = values.map do |value|
+      if previous
+        delta = (value - previous).abs
+        volatility = delta
+      end
+      previous = value
+      volatility
+    end
+    #deltas => [nil, 1, 1]
+    deltas.shift
+    volatility = deltas.sum
+  end
+  def self.process_values(map)
+    map.transform_values(&method(:process))
+  end
 end
 
-ActiveRecord::Base.establish_connection ENV["PG_URI"]
 ActiveRecord::Base.connection.add_toolkit_to_search_path!
 
 
@@ -58,7 +93,12 @@ if Measurement.count.zero?
      SQL
 end
 
+
+volatilities = nil
+#ActiveRecord::Base.logger = nil
 Benchmark.bm do |x|
-  x.report("ruby")  { Measurement.volatility_ruby }
   x.report("sql") { Measurement.volatility_sql.map(&:attributes)  }
+  x.report("ruby")  { Measurement.volatility_ruby }
+  x.report("fetch") { volatilities =  Measurement.values_from_devices }
+  x.report("process") { Volatility.process_values(volatilities) }
 end

data/examples/toolkit-demo/ohlc.rb

@@ -0,0 +1,175 @@
+# ruby ohlc.rb postgres://user:pass@host:port/db_name
+# @see https://jonatas.github.io/timescaledb/ohlc_tutorial
+
+require 'bundler/inline' #require only what you need
+
+gemfile(true) do
+  gem 'timescaledb', path:  '../..'
+  gem 'pry'
+end
+
+ActiveRecord::Base.establish_connection ARGV.last
+
+# Compare ohlc processing in Ruby vs SQL.
+class Tick < ActiveRecord::Base
+  acts_as_hypertable time_column: "time"
+  acts_as_time_vector segment_by: "symbol", value_column: "price"
+end
+require "active_support/concern"
+
+module Ohlc
+  extend ActiveSupport::Concern
+
+  included do
+    %w[open high low close].each do |name|
+      attribute name, :decimal
+      attribute "#{name}_time", :time
+    end
+
+
+    scope :attributes, -> do
+      select("symbol, time,
+        toolkit_experimental.open(ohlc),
+        toolkit_experimental.high(ohlc),
+        toolkit_experimental.low(ohlc),
+        toolkit_experimental.close(ohlc),
+        toolkit_experimental.open_time(ohlc),
+        toolkit_experimental.high_time(ohlc),
+        toolkit_experimental.low_time(ohlc),
+        toolkit_experimental.close_time(ohlc)")
+    end
+
+    scope :rollup, -> (timeframe: '1h') do
+      select("symbol, time_bucket('#{timeframe}', time) as time,
+            toolkit_experimental.rollup(ohlc) as ohlc")
+      .group(1,2)
+    end
+
+    def readonly?
+      true
+    end
+  end
+
+  class_methods do
+  end
+end
+
+class Ohlc1m < ActiveRecord::Base
+  self.table_name = 'ohlc_1m'
+  include Ohlc
+end
+
+class Ohlc1h < ActiveRecord::Base
+  self.table_name = 'ohlc_1h'
+  include Ohlc
+end
+
+class Ohlc1d < ActiveRecord::Base
+  self.table_name = 'ohlc_1d'
+  include Ohlc
+end
+=begin
+  scope :ohlc_ruby, -> (
+    timeframe: 1.hour,
+    segment_by: segment_by_column,
+    time: time_column,
+    value: value_column) {
+    ohlcs = Hash.new() {|hash, key| hash[key] = [] }
+
+    key = tick.send(segment_by)
+    candlestick = ohlcs[key].last
+    if candlestick.nil? || candlestick.time + timeframe > tick.time
+      ohlcs[key] << Candlestick.new(time $, price)
+    end
+    find_all do |tick|
+      symbol = tick.symbol
+
+      if previous[symbol]
+        delta = (tick.price - previous[symbol]).abs
+        volatility[symbol] += delta
+      end
+      previous[symbol] = tick.price
+    end
+    volatility
+  }
+=end
+
+ActiveRecord::Base.connection.add_toolkit_to_search_path!
+
+
+ActiveRecord::Base.connection.instance_exec do
+  ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+  unless Tick.table_exists?
+    hypertable_options = {
+      time_column: 'time',
+      chunk_time_interval: '1 week',
+      compress_segmentby: 'symbol',
+      compress_orderby: 'time',
+      compression_interval: '1 month'
+    }
+    create_table :ticks, hypertable: hypertable_options, id: false do |t|
+      t.column :time , 'timestamp with time zone'
+      t.string :symbol
+      t.decimal :price
+      t.integer :volume
+    end
+
+    options = {
+      with_data: false,
+      refresh_policies: {
+        start_offset: "INTERVAL '1 month'",
+        end_offset: "INTERVAL '1 minute'",
+        schedule_interval: "INTERVAL '1 minute'"
+      }
+    }
+    create_continuous_aggregate('ohlc_1m', Tick._ohlc(timeframe: '1m'), **options)
+
+    execute "CREATE VIEW ohlc_1h AS #{ Ohlc1m.rollup(timeframe: '1 hour').to_sql}"
+    execute "CREATE VIEW ohlc_1d AS #{ Ohlc1h.rollup(timeframe: '1 day').to_sql}"
+  end
+end
+
+if Tick.count.zero?
+  ActiveRecord::Base.connection.execute(<<~SQL)
+    INSERT INTO ticks
+    SELECT time, 'SYMBOL', 1 + (random()*30)::int, 100*(random()*10)::int
+    FROM generate_series(TIMESTAMP '2022-01-01 00:00:00',
+                    TIMESTAMP '2022-02-01 00:01:00',
+                INTERVAL '1 second') AS time;
+     SQL
+end
+
+
+# Fetch attributes
+Ohlc1m.attributes
+
+# Rollup demo
+
+# Attributes from rollup
+Ohlc1m.attributes.from(Ohlc1m.rollup(timeframe: '1 day'))
+
+
+# Nesting several levels
+Ohlc1m.attributes.from(
+  Ohlc1m.rollup(timeframe: '1 week').from(
+    Ohlc1m.rollup(timeframe: '1 day')
+  )
+)
+Ohlc1m.attributes.from(
+  Ohlc1m.rollup(timeframe: '1 month').from(
+    Ohlc1m.rollup(timeframe: '1 week').from(
+      Ohlc1m.rollup(timeframe: '1 day')
+    )
+  )
+)
+
+Pry.start
+
+=begin
+TODO: implement the ohlc_ruby 
+Benchmark.bm do |x|
+  x.report("ruby")  { Tick.ohlc_ruby }
+  x.report("sql") { Tick.ohlc.map(&:attributes)  }
+end
+=end

data/lib/timescaledb/dimensions.rb

@@ -1,7 +1,7 @@
 module Timescaledb
   class Dimension < ActiveRecord::Base
     self.table_name = "timescaledb_information.dimensions"
-    attribute :time_interval, :interval
+#    attribute :time_interval, :interval
   end
   Dimensions = Dimension
 end

data/lib/timescaledb/job_stats.rb

@@ -3,6 +3,7 @@ module Timescaledb
     self.table_name = "timescaledb_information.job_stats"
 
     belongs_to :job
+#    attribute :last_run_duration, :interval
 
     scope :success, -> { where(last_run_status: "Success") }
     scope :scheduled, -> { where(job_status: "Scheduled") }

data/lib/timescaledb/migration_helpers.rb

@@ -80,7 +80,7 @@ module Timescaledb
         WITH #{"NO" unless options[:with_data]} DATA;
       SQL
 
-      create_continuous_aggregate_policy(table_name, options[:refresh_policies] || {})
+      create_continuous_aggregate_policy(table_name, **(options[:refresh_policies] || {}))
     end
 
 

data/lib/timescaledb/schema_dumper.rb

@@ -6,15 +6,11 @@ module Timescaledb
     def tables(stream)
       super # This will call #table for each table in the database
       views(stream) unless defined?(Scenic) # Don't call this twice if we're using Scenic
-    end
 
-    def table(table_name, stream)
-      super(table_name, stream)
-      if Timescaledb::Hypertable.table_exists? &&
-         (hypertable = Timescaledb::Hypertable.find_by(hypertable_name: table_name))
-        timescale_hypertable(hypertable, stream)
-        timescale_retention_policy(hypertable, stream)
-      end
+      return unless Timescaledb::Hypertable.table_exists?
+
+      timescale_hypertables(stream)
+      timescale_retention_policies(stream)
     end
 
     def views(stream)
@@ -24,23 +20,37 @@ module Timescaledb
       super if defined?(super)
     end
 
+    def timescale_hypertables(stream)
+      stream.puts # Insert a blank line above the hypertable definitions, for readability
+
+      sorted_hypertables.each do |hypertable|
+         timescale_hypertable(hypertable, stream)
+      end
+    end
+
+    def timescale_retention_policies(stream)
+      stream.puts # Insert a blank line above the retention policies, for readability
+
+      sorted_hypertables.each do |hypertable|
+        timescale_retention_policy(hypertable, stream)
+      end
+    end
+
     private
 
     def timescale_hypertable(hypertable, stream)
-      dim = hypertable.dimensions
+      dim = hypertable.main_dimension
       extra_settings = {
         time_column: "#{dim.column_name}",
         chunk_time_interval: "#{dim.time_interval.inspect}"
       }.merge(timescale_compression_settings_for(hypertable)).map {|k, v| %Q[#{k}: "#{v}"]}.join(", ")
 
       stream.puts %Q[  create_hypertable "#{hypertable.hypertable_name}", #{extra_settings}]
-      stream.puts
     end
 
     def timescale_retention_policy(hypertable, stream)
       hypertable.jobs.where(proc_name: "policy_retention").each do |job|
         stream.puts %Q[  create_retention_policy "#{job.hypertable_name}", interval: "#{job.config["drop_after"]}"]
-        stream.puts
       end
     end
 
@@ -85,6 +95,9 @@ module Timescaledb
 
       "INTERVAL '#{value}'"
     end
+    def sorted_hypertables
+      @sorted_hypertables ||= Timescaledb::Hypertable.order(:hypertable_name).to_a
+    end
   end
 end
 

data/lib/timescaledb/toolkit/time_vector.rb

@@ -13,8 +13,9 @@ module Timescaledb
         end
 
         def time_column
-          respond_to?(:time_column) && super || time_vector_options[:time_column] 
+          respond_to?(:time_column) && super || time_vector_options[:time_column]
         end
+
         def segment_by_column
           time_vector_options[:segment_by]
         end
@@ -25,8 +26,7 @@ module Timescaledb
           scope :volatility, -> (segment_by: segment_by_column) do
             select([*segment_by,
                "timevector(#{time_column}, #{value_column}) -> sort() -> delta() -> abs() -> sum() as volatility"
-            ].join(", "))
-              .group(segment_by)
+            ].join(", ")).group(segment_by)
           end
 
           scope :time_weight, -> (segment_by: segment_by_column) do
@@ -40,8 +40,7 @@ module Timescaledb
             lttb_query = <<~SQL
               WITH x AS ( #{select(*segment_by, time_column, value_column).to_sql})
               SELECT #{"x.#{segment_by}," if segment_by}
-                (lttb( x.#{time_column}, x.#{value_column}, #{threshold})
-                 -> toolkit_experimental.unnest()).*
+                (lttb( x.#{time_column}, x.#{value_column}, #{threshold}) -> unnest()).*
               FROM x
               #{"GROUP BY device_id" if segment_by}
             SQL
@@ -58,6 +57,38 @@ module Timescaledb
               downsampled.map{|e|[ e[time_column],e[value_column]]}
             end
           end
+
+
+          scope :_ohlc, -> (timeframe: '1h',
+                           segment_by: segment_by_column,
+                           time: time_column,
+                           value: value_column) do
+
+             select( "time_bucket('#{timeframe}', #{time}) as #{time}",
+                           *segment_by,
+                           "toolkit_experimental.ohlc(#{time}, #{value})")
+              .order(1)
+              .group(*(segment_by ? [1,2] : 1))
+          end
+
+          scope :ohlc, -> (timeframe: '1h',
+                           segment_by: segment_by_column,
+                           time: time_column,
+                           value: value_column) do
+
+            raw = _ohlc(timeframe: timeframe, segment_by: segment_by, time: time, value: value)
+            unscoped
+              .from("(#{raw.to_sql}) AS ohlc")
+              .select(*segment_by, time,
+               "toolkit_experimental.open(ohlc),
+                toolkit_experimental.high(ohlc),
+                toolkit_experimental.low(ohlc),
+                toolkit_experimental.close(ohlc),
+                toolkit_experimental.open_time(ohlc),
+                toolkit_experimental.high_time(ohlc),
+                toolkit_experimental.low_time(ohlc),
+                toolkit_experimental.close_time(ohlc)")
+          end
         end
       end
     end

data/lib/timescaledb/version.rb

@@ -1,3 +1,3 @@
 module Timescaledb
-  VERSION = '0.2.2'
+  VERSION = '0.2.4'
 end

data/mkdocs.yml

@@ -29,5 +29,6 @@ nav:
   - Toolkit Integration: toolkit.md
   - Toolkit LTTB Tutorial: toolkit_lttb_tutorial.md
   - Zooming with High Resolution: toolkit_lttb_zoom.md
+  - Toolkit OHLC: toolkit_ohlc.md
   - Command Line: command_line.md
   - Videos: videos.md

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timescaledb
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - Jônatas Davi Paganini
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-14 00:00:00.000000000 Z
+date: 2022-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -150,6 +150,7 @@ files:
 - ".tool-versions"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
+- Fastfile
 - Gemfile
 - Gemfile.lock
 - Gemfile.scenic
@@ -170,6 +171,7 @@ files:
 - docs/toolkit.md
 - docs/toolkit_lttb_tutorial.md
 - docs/toolkit_lttb_zoom.md
+- docs/toolkit_ohlc.md
 - docs/videos.md
 - examples/all_in_one/all_in_one.rb
 - examples/all_in_one/benchmark_comparison.rb
@@ -233,6 +235,7 @@ files:
 - examples/toolkit-demo/lttb/lttb_sinatra.rb
 - examples/toolkit-demo/lttb/lttb_test.rb
 - examples/toolkit-demo/lttb/views/index.erb
+- examples/toolkit-demo/ohlc.rb
 - lib/timescaledb.rb
 - lib/timescaledb/acts_as_hypertable.rb
 - lib/timescaledb/acts_as_hypertable/core.rb
@@ -261,7 +264,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/jonatas/timescaledb
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -276,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: TimescaleDB helpers for Ruby ecosystem.
 test_files: []