active_hll 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a9014f39ae62f6db1066d4933bc28485ecc57e788922d70437c4744edfb0c033
+  data.tar.gz: 91320d88909c41e425966993e145b9ff8aa5b57c6c3ac0556dc3661519691968
+SHA512:
+  metadata.gz: c55980654230a259a249a24b959c18b054d8e41773653bcc5411dbbbe3200fdd35c65dcfe729ed042943a8f2e81e26843c696d4989840fa43384622e3d56614f
+  data.tar.gz: 5285ca0005787997dec664512b0bc6796201a8a4bd719e0ce0b9f6efa50096f3a3e7ade096db6ba943e9f84630b2a92c5d9b3a53a4a0b744cc4162e87d6c2990

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2023-01-24)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,157 @@
+# Active HLL
+
+:fire: HyperLogLog for Rails and Postgres
+
+For fast, approximate count-distinct queries
+
+[![Build Status](https://github.com/ankane/active_hll/workflows/build/badge.svg?branch=master)](https://github.com/ankane/active_hll/actions)
+
+## Installation
+
+First, install the [hll extension](https://github.com/citusdata/postgresql-hll) on your database server:
+
+```sh
+cd /tmp
+curl -L https://github.com/citusdata/postgresql-hll/archive/refs/tags/v2.17.tar.gz | tar xz
+cd postgresql-hll-2.17
+make
+make install # may need sudo
+```
+
+Then add this line to your application’s Gemfile:
+
+```ruby
+gem "active_hll"
+```
+
+And run:
+
+```sh
+bundle install
+rails generate active_hll:install
+rails db:migrate
+```
+
+## Getting Started
+
+HLLs provide an approximate count of unique values (like unique visitors). By rolling up data by day, you can quickly get an approximate count over any date range.
+
+Create a table with an `hll` column
+
+```ruby
+class CreateEventRollups < ActiveRecord::Migration[7.0]
+  def change
+    create_table :event_rollups do |t|
+      t.date :time_bucket, index: {unique: true}
+      t.hll :visitor_ids
+    end
+  end
+end
+```
+
+You can use [batch](#batch) and [stream](#stream) approaches to build HLLs
+
+### Batch
+
+Use the `hll_agg` method to generate HLLs from existing data
+
+```ruby
+hlls = Event.group_by_day(:created_at).hll_agg(:visitor_id)
+```
+
+> Install [Groupdate](https://github.com/ankane/groupdate) to use the `group_by_day` method
+
+And store the result
+
+```ruby
+EventRollup.upsert_all(
+  hlls.map { |k, v| {time_bucket: k, visitor_ids: v} },
+  unique_by: [:time_bucket]
+)
+```
+
+For a large number of HLLs, use SQL to generate and upsert in a single statement
+
+### Stream
+
+Use the `hll_add` method to add new data to HLLs
+
+```ruby
+EventRollup.where(time_bucket: Date.current).hll_add(visitor_ids: ["visitor1", "visitor2"])
+```
+
+## Querying
+
+Get approximate unique values for a time range
+
+```ruby
+EventRollup.where(time_bucket: 30.days.ago.to_date..Date.current).hll_count(:visitor_ids)
+```
+
+Get approximate unique values by time bucket
+
+```ruby
+EventRollup.group(:time_bucket).hll_count(:visitor_ids)
+```
+
+Get approximate unique values by month
+
+```ruby
+EventRollup.group_by_month(:time_bucket, time_zone: false).hll_count(:visitor_ids)
+```
+
+Get the union of multiple HLLs
+
+```ruby
+EventRollup.hll_union(:visitor_ids)
+```
+
+## Data Protection
+
+Cardinality estimators like HyperLogLog do not [preserve privacy](https://arxiv.org/pdf/1808.05879.pdf), so protect `hll` columns the same as you would the raw data.
+
+For instance, you can check membership with a good probability with:
+
+```sql
+SELECT
+    time_bucket,
+    visitor_ids = visitor_ids || hll_hash_text('visitor1') AS likely_member
+FROM
+    event_rollups;
+```
+
+## Data Retention
+
+Data should only be retained for as long as it’s needed. Delete older data with:
+
+```ruby
+EventRollup.where("time_bucket < ?", 2.years.ago).delete_all
+```
+
+There’s not a way to remove data from an HLL, so to delete data for a specific user, delete the underlying data and recalculate the rollup.
+
+## Hosted Postgres
+
+The `hll` extension is available on Amazon RDS, Google Cloud SQL, and DigitalOcean Managed Databases.
+
+## History
+
+View the [changelog](CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/active_hll/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/active_hll/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/active_hll.git
+cd active_hll
+bundle install
+bundle exec rake test
+```

data/lib/active_hll/hll.rb

@@ -0,0 +1,51 @@
+# format of value
+# https://github.com/aggregateknowledge/hll-storage-spec/blob/v1.0.0/STORAGE.md
+module ActiveHll
+  class Hll
+    attr_reader :value
+
+    def initialize(value)
+      unless value.is_a?(String) && value.encoding == Encoding::BINARY
+        raise ArgumentError, "Expected binary string"
+      end
+
+      @value = value
+    end
+
+    def inspect
+      "(hll)"
+    end
+
+    def schema_version
+      value[0].unpack1("C") >> 4
+    end
+
+    def type
+      value[0].unpack1("C") & 0b00001111
+    end
+
+    def regwidth
+      (value[1].unpack1("C") >> 5) + 1
+    end
+
+    def log2m
+      value[1].unpack1("C") & 0b00011111
+    end
+
+    def sparseon
+      (value[2].unpack1("C") & 0b01000000) >> 6
+    end
+
+    def expthresh
+      t = value[2].unpack1("C") & 0b00111111
+      t == 63 ? -1 : 2**(t - 1)
+    end
+
+    def data
+      case type
+      when 2
+        value[3..-1].unpack("q>*")
+      end
+    end
+  end
+end

data/lib/active_hll/model.rb

@@ -0,0 +1,66 @@
+require "active_support/concern"
+
+module ActiveHll
+  module Model
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def hll_agg(column)
+        Utils.hll_calculate(self, "hll_add_agg(hll_hash_any(%s)) AS hll_agg", column, default_value: nil)
+      end
+
+      def hll_union(column)
+        Utils.hll_calculate(self, "hll_union_agg(%s) AS hll_union", column, default_value: nil)
+      end
+
+      def hll_count(column)
+        Utils.hll_calculate(self, "hll_cardinality(hll_union_agg(%s)) AS hll_count", column, default_value: 0.0)
+      end
+
+      # experimental
+      # doesn't work with non-default parameters
+      def hll_generate(values)
+        parts = ["hll_empty()"]
+
+        values.each do |value|
+          parts << Utils.hll_hash_sql(self, value)
+        end
+
+        result = connection.select_all("SELECT #{parts.join(" || ")}").rows[0][0]
+        ActiveHll::Type.new.deserialize(result)
+      end
+
+      def hll_add(attributes)
+        set_clauses =
+          attributes.map do |attribute, values|
+            values = [values] unless values.is_a?(Array)
+            return 0 if values.empty?
+
+            quoted_column = connection.quote_column_name(attribute)
+            # possibly fetch parameters for the column in the future
+            # for now, users should set a default value on the column
+            parts = ["COALESCE(#{quoted_column}, hll_empty())"]
+
+            values.each do |value|
+              parts << Utils.hll_hash_sql(self, value)
+            end
+
+            "#{quoted_column} = #{parts.join(" || ")}"
+          end
+
+        update_all(set_clauses.join(", "))
+      end
+    end
+
+    # doesn't update in-memory record attribute for performance
+    def hll_add(attributes)
+      self.class.where(id: id).hll_add(attributes)
+      nil
+    end
+
+    def hll_count(attribute)
+      quoted_column = self.class.connection.quote_column_name(attribute)
+      self.class.where(id: id).pluck("hll_cardinality(#{quoted_column})").first || 0.0
+    end
+  end
+end

data/lib/active_hll/type.rb

@@ -0,0 +1,21 @@
+module ActiveHll
+  class Type < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+    def type
+      :hll
+    end
+
+    def serialize(value)
+      if value.is_a?(Hll)
+        value = value.value
+      elsif !value.nil?
+        raise ArgumentError, "can't cast #{value.class.name} to hll"
+      end
+      super(value)
+    end
+
+    def deserialize(value)
+      value = super
+      value.nil? ? value : Hll.new(value)
+    end
+  end
+end

data/lib/active_hll/utils.rb

@@ -0,0 +1,70 @@
+module ActiveHll
+  module Utils
+    class << self
+      def hll_hash_sql(klass, value)
+        hash_function =
+          case value
+          when true, false
+            "hll_hash_boolean"
+          when Integer
+            "hll_hash_bigint"
+          when String
+            "hll_hash_text"
+          else
+            raise ArgumentError, "Unexpected type: #{value.class.name}"
+          end
+        quoted_value = klass.connection.quote(value)
+        "#{hash_function}(#{quoted_value})"
+      end
+
+      def hll_calculate(relation, operation, column, default_value:)
+        sql, relation, group_values = hll_calculate_sql(relation, operation, column)
+        result = relation.connection.select_all(sql)
+
+        # typecast
+        rows = []
+        columns = result.columns
+        result.rows.each do |untyped_row|
+          rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].deserialize(untyped_row[i]) : untyped_row[i] })
+        end
+
+        result =
+          if group_values.any?
+            Hash[rows.map { |r| [r.size == 2 ? r[0] : r[0..-2], r[-1]] }]
+          else
+            rows[0] && rows[0][0]
+          end
+
+        result = Groupdate.process_result(relation, result, default_value: default_value) if defined?(Groupdate.process_result)
+
+        result
+      end
+
+      def hll_calculate_sql(relation, operation, column)
+        # basic version of Active Record disallow_raw_sql!
+        # symbol = column (safe), Arel node = SQL (safe), other = untrusted
+        # matches table.column and column
+        unless column.is_a?(Symbol) || column.is_a?(Arel::Nodes::SqlLiteral)
+          column = column.to_s
+          unless /\A\w+(\.\w+)?\z/i.match(column)
+            raise ActiveRecord::UnknownAttributeReference, "Query method called with non-attribute argument(s): #{column.inspect}. Use Arel.sql() for known-safe values."
+          end
+        end
+
+        # column resolution
+        node = relation.all.send(:arel_columns, [column]).first
+        node = Arel::Nodes::SqlLiteral.new(node) if node.is_a?(String)
+        column = relation.connection.visitor.accept(node, Arel::Collectors::SQLString.new).value
+
+        group_values = relation.all.group_values
+
+        relation = relation.unscope(:select).select(*group_values, operation % [column])
+
+        # same as average
+        relation = relation.unscope(:order).distinct!(false) if group_values.empty?
+
+        [relation.to_sql, relation, group_values]
+      end
+    end
+  end
+end

data/lib/active_hll/version.rb

@@ -0,0 +1,3 @@
+module ActiveHll
+  VERSION = "0.1.0"
+end

data/lib/active_hll.rb

@@ -0,0 +1,41 @@
+# dependencies
+require "active_support"
+
+# modules
+require_relative "active_hll/hll"
+require_relative "active_hll/utils"
+require_relative "active_hll/version"
+
+module ActiveHll
+  class Error < StandardError; end
+
+  autoload :Type, "active_hll/type"
+
+  module RegisterType
+    def initialize_type_map(m = type_map)
+      super
+      m.register_type "hll", ActiveHll::Type.new
+    end
+  end
+end
+
+ActiveSupport.on_load(:active_record) do
+  require_relative "active_hll/model"
+
+  include ActiveHll::Model
+
+  require "active_record/connection_adapters/postgresql_adapter"
+
+  # ensure schema can be dumped
+  ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::NATIVE_DATABASE_TYPES[:hll] = {name: "hll"}
+
+  # ensure schema can be loaded
+  ActiveRecord::ConnectionAdapters::TableDefinition.send(:define_column_methods, :hll)
+
+  # prevent unknown OID warning
+  if ActiveRecord::VERSION::MAJOR >= 7
+    ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.singleton_class.prepend(ActiveHll::RegisterType)
+  else
+    ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveHll::RegisterType)
+  end
+end

data/lib/generators/active_hll/install_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module ActiveHll
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "migration.rb", "db/migrate/install_active_hll.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/active_hll/templates/migration.rb.tt

@@ -0,0 +1,5 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    enable_extension "hll"
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: active_hll
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-01-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+description:
+email: andrew@ankane.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/active_hll.rb
+- lib/active_hll/hll.rb
+- lib/active_hll/model.rb
+- lib/active_hll/type.rb
+- lib/active_hll/utils.rb
+- lib/active_hll/version.rb
+- lib/generators/active_hll/install_generator.rb
+- lib/generators/active_hll/templates/migration.rb.tt
+homepage: https://github.com/ankane/active_hll
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key:
+specification_version: 4
+summary: HyperLogLog for Rails and Postgres
+test_files: []