logidze 0.8.1 → 1.0.0.rc1

data/lib/generators/logidze/fx_helper.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Logidze
+  module Generators
+    # Adds --fx option and provide #fx? method
+    module FxHelper
+      def self.included(base)
+        base.class_option :fx, type: :boolean, optional: true,
+                               desc: "Define whether to use fx gem functionality"
+      end
+
+      def fx?
+        options[:fx] || (options[:fx] != false && defined?(::Fx::SchemaDumper))
+      end
+    end
+  end
+end

data/lib/generators/logidze/inject_sql.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Logidze
+  module Generators
+    module InjectSql
+      def inject_sql(source, indent: 4)
+        source = ::File.expand_path(find_in_source_paths(source.to_s))
+
+        indent(
+          ERB.new(::File.binread(source)).tap do |erb|
+            erb.filename = source
+          end.result(instance_eval("binding")), # rubocop:disable Style/EvalWithLocation
+          indent
+        )
+      end
+    end
+  end
+end

data/lib/generators/logidze/install/USAGE

@@ -1,7 +1,12 @@
 Description:
   Generates the necessary files to get you up and running with Logidze gem
-  
+
 Examples:
   rails generate logidze:install
 
     This will generate the core migration file with trigger function defined.
+
+  rails generate logidze:install --fx
+
+    This will generate schema.rb compatible migration with `create_function` definitions and separate SQL files.
+    The fx gem must be installed.

data/lib/generators/logidze/install/functions/logidze_compact_history.sql

@@ -0,0 +1,38 @@
+-- version: 1
+CREATE OR REPLACE FUNCTION logidze_compact_history(log_data jsonb, cutoff integer DEFAULT 1) RETURNS jsonb AS $body$
+  DECLARE
+    merged jsonb;
+  BEGIN
+    LOOP
+      merged := jsonb_build_object(
+        'ts',
+        log_data#>'{h,1,ts}',
+        'v',
+        log_data#>'{h,1,v}',
+        'c',
+        (log_data#>'{h,0,c}') || (log_data#>'{h,1,c}')
+      );
+
+      IF (log_data#>'{h,1}' ? 'm') THEN
+        merged := jsonb_set(merged, ARRAY['m'], log_data#>'{h,1,m}');
+      END IF;
+
+      log_data := jsonb_set(
+        log_data,
+        '{h}',
+        jsonb_set(
+          log_data->'h',
+          '{1}',
+          merged
+        ) - 0
+      );
+
+      cutoff := cutoff - 1;
+
+      EXIT WHEN cutoff <= 0;
+    END LOOP;
+
+    return log_data;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_filter_keys.sql

@@ -0,0 +1,27 @@
+-- version: 1
+CREATE OR REPLACE FUNCTION logidze_filter_keys(obj jsonb, keys text[], include_columns boolean DEFAULT false) RETURNS jsonb AS $body$
+  DECLARE
+    res jsonb;
+    key text;
+  BEGIN
+    res := '{}';
+
+    IF include_columns THEN
+      FOREACH key IN ARRAY keys
+      LOOP
+        IF obj ? key THEN
+          res = jsonb_insert(res, ARRAY[key], obj->key);
+        END IF;
+      END LOOP;
+    ELSE
+      res = obj;
+      FOREACH key IN ARRAY keys
+      LOOP
+        res = res - key;
+      END LOOP;
+    END IF;
+
+    RETURN res;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_logger.sql

@@ -0,0 +1,150 @@
+-- version: 1
+CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
+  DECLARE
+    changes jsonb;
+    version jsonb;
+    snapshot jsonb;
+    new_v integer;
+    size integer;
+    history_limit integer;
+    debounce_time integer;
+    current_version integer;
+    merged jsonb;
+    iterator integer;
+    item record;
+    columns text[];
+    include_columns boolean;
+    ts timestamp with time zone;
+    ts_column text;
+  BEGIN
+    ts_column := NULLIF(TG_ARGV[1], 'null');
+    columns := NULLIF(TG_ARGV[2], 'null');
+    include_columns := NULLIF(TG_ARGV[3], 'null');
+
+    IF TG_OP = 'INSERT' THEN
+      -- always exclude log_data column
+      changes := to_jsonb(NEW.*) - 'log_data';
+
+      IF columns IS NOT NULL THEN
+        snapshot = logidze_snapshot(changes, ts_column, columns, include_columns);
+      ELSE
+        snapshot = logidze_snapshot(changes, ts_column);
+      END IF;
+
+      IF snapshot#>>'{h, -1, c}' != '{}' THEN
+        NEW.log_data := snapshot;
+      END IF;
+
+    ELSIF TG_OP = 'UPDATE' THEN
+
+      IF OLD.log_data is NULL OR OLD.log_data = '{}'::jsonb THEN
+        -- always exclude log_data column
+        changes := to_jsonb(NEW.*) - 'log_data';
+
+        IF columns IS NOT NULL THEN
+          snapshot = logidze_snapshot(changes, ts_column, columns, include_columns);
+        ELSE
+          snapshot = logidze_snapshot(changes, ts_column);
+        END IF;
+
+        IF snapshot#>>'{h, -1, c}' != '{}' THEN
+          NEW.log_data := snapshot;
+        END IF;
+        RETURN NEW;
+      END IF;
+
+      history_limit := NULLIF(TG_ARGV[0], 'null');
+      debounce_time := NULLIF(TG_ARGV[4], 'null');
+
+      current_version := (NEW.log_data->>'v')::int;
+
+      IF ts_column IS NULL THEN
+        ts := statement_timestamp();
+      ELSE
+        ts := (to_jsonb(NEW.*)->>ts_column)::timestamp with time zone;
+        IF ts IS NULL OR ts = (to_jsonb(OLD.*)->>ts_column)::timestamp with time zone THEN
+          ts := statement_timestamp();
+        END IF;
+      END IF;
+
+      IF NEW = OLD THEN
+        RETURN NEW;
+      END IF;
+
+      IF current_version < (NEW.log_data#>>'{h,-1,v}')::int THEN
+        iterator := 0;
+        FOR item in SELECT * FROM jsonb_array_elements(NEW.log_data->'h')
+        LOOP
+          IF (item.value->>'v')::int > current_version THEN
+            NEW.log_data := jsonb_set(
+              NEW.log_data,
+              '{h}',
+              (NEW.log_data->'h') - iterator
+            );
+          END IF;
+          iterator := iterator + 1;
+        END LOOP;
+      END IF;
+
+      changes := '{}';
+
+      IF (coalesce(current_setting('logidze.full_snapshot', true), '') = 'on') THEN
+        changes = hstore_to_jsonb_loose(hstore(NEW.*));
+      ELSE
+        changes = hstore_to_jsonb_loose(
+          hstore(NEW.*) - hstore(OLD.*)
+        );
+      END IF;
+
+      changes = changes - 'log_data';
+
+      IF columns IS NOT NULL THEN
+        changes = logidze_filter_keys(changes, columns, include_columns);
+      END IF;
+
+      IF changes = '{}' THEN
+        RETURN NEW;
+      END IF;
+
+      new_v := (NEW.log_data#>>'{h,-1,v}')::int + 1;
+
+      size := jsonb_array_length(NEW.log_data->'h');
+      version := logidze_version(new_v, changes, ts);
+
+      IF (
+        debounce_time IS NOT NULL AND
+        (version->>'ts')::bigint - (NEW.log_data#>'{h,-1,ts}')::text::bigint <= debounce_time
+      ) THEN
+        -- merge new version with the previous one
+        new_v := (NEW.log_data#>>'{h,-1,v}')::int;
+        version := logidze_version(new_v, (NEW.log_data#>'{h,-1,c}')::jsonb || changes, ts);
+        -- remove the previous version from log
+        NEW.log_data := jsonb_set(
+          NEW.log_data,
+          '{h}',
+          (NEW.log_data->'h') - (size - 1)
+        );
+      END IF;
+
+      NEW.log_data := jsonb_set(
+        NEW.log_data,
+        ARRAY['h', size::text],
+        version,
+        true
+      );
+
+      NEW.log_data := jsonb_set(
+        NEW.log_data,
+        '{v}',
+        to_jsonb(new_v)
+      );
+
+      IF history_limit IS NOT NULL AND history_limit <= size THEN
+        NEW.log_data := logidze_compact_history(NEW.log_data, size - history_limit + 1);
+      END IF;
+    END IF;
+
+    return NEW;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_snapshot.sql

@@ -0,0 +1,24 @@
+-- version: 1
+CREATE OR REPLACE FUNCTION logidze_snapshot(item jsonb, ts_column text DEFAULT NULL, columns text[] DEFAULT NULL, include_columns boolean DEFAULT false) RETURNS jsonb AS $body$
+  DECLARE
+    ts timestamp with time zone;
+  BEGIN
+    IF ts_column IS NULL THEN
+      ts := statement_timestamp();
+    ELSE
+      ts := coalesce((item->>ts_column)::timestamp with time zone, statement_timestamp());
+    END IF;
+
+    IF columns IS NOT NULL THEN
+      item := logidze_filter_keys(item, columns, include_columns);
+    END IF;
+
+    return json_build_object(
+      'v', 1,
+      'h', jsonb_build_array(
+              logidze_version(1, item, ts)
+            )
+      );
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_version.sql

@@ -0,0 +1,20 @@
+-- version: 1
+CREATE OR REPLACE FUNCTION logidze_version(v bigint, data jsonb, ts timestamp with time zone) RETURNS jsonb AS $body$
+  DECLARE
+    buf jsonb;
+  BEGIN
+    buf := jsonb_build_object(
+              'ts',
+              (extract(epoch from ts) * 1000)::bigint,
+              'v',
+              v,
+              'c',
+              data
+              );
+    IF coalesce(current_setting('logidze.meta', true), '') <> '' THEN
+      buf := jsonb_insert(buf, '{m}', current_setting('logidze.meta')::jsonb);
+    END IF;
+    RETURN buf;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/install_generator.rb

@@ -1,19 +1,30 @@
 # frozen_string_literal: true
+
 require "rails/generators"
 require "rails/generators/active_record"
+require_relative "../inject_sql"
+require_relative "../fx_helper"
+
+using RubyNext
 
 module Logidze
   module Generators
     class InstallGenerator < ::Rails::Generators::Base # :nodoc:
       include Rails::Generators::Migration
+      include InjectSql
+      include FxHelper
+
+      class FuncDef < Struct.new(:name, :version, :signature); end
 
-      source_root File.expand_path('templates', __dir__)
+      source_root File.expand_path("templates", __dir__)
+      source_paths << File.expand_path("functions", __dir__)
 
       class_option :update, type: :boolean, optional: true,
                             desc: "Define whether this is an update migration"
 
       def generate_migration
-        migration_template "migration.rb.erb", "db/migrate/#{migration_name}.rb"
+        migration_template = fx? ? "migration_fx.rb.erb" : "migration.rb.erb"
+        migration_template migration_template, "db/migrate/#{migration_name}.rb"
       end
 
       def generate_hstore_migration
@@ -22,10 +33,20 @@ module Logidze
         migration_template "hstore.rb.erb", "db/migrate/enable_hstore.rb"
       end
 
+      def generate_fx_functions
+        return unless fx?
+
+        function_definitions.each do |fdef|
+          next if fdef.version == previous_version_for(fdef.name)
+
+          template "#{fdef.name}.sql", "db/functions/#{fdef.name}_v#{fdef.version.to_s.rjust(2, "0")}.sql"
+        end
+      end
+
       no_tasks do
         def migration_name
           if update?
-            "logidze_update_#{Logidze::VERSION.delete('.')}"
+            "logidze_update_#{Logidze::VERSION.delete(".")}"
           else
             "logidze_install"
           end
@@ -38,6 +59,43 @@ module Logidze
         def update?
           options[:update]
         end
+
+        def previous_version_for(name)
+          all_functions.filter_map { |path| Regexp.last_match[1].to_i if path =~ %r{#{name}_v(\d+).sql} }.max
+        end
+
+        def all_functions
+          @all_functions ||=
+            begin
+              res = nil
+              in_root do
+                res = if File.directory?("db/functions")
+                  Dir.entries("db/functions")
+                else
+                  []
+                end
+              end
+              res
+            end
+        end
+
+        def function_definitions
+          @function_definitions ||=
+            begin
+              Dir.glob(File.join(__dir__, "functions", "*.sql")).map do |path|
+                name = path.match(/([^\/]+)\.sql/)[1]
+
+                file = File.open(path)
+                header = file.readline
+
+                version = header.match(/version:\s+(\d+)/)[1].to_i
+                parameters = file.readline.match(/CREATE OR REPLACE FUNCTION\s+[\w_]+\((.*)\)/)[1]
+                signature = parameters.split(/\s*,\s*/).map { |param| param.split(/\s+/, 2).last.sub(/\s+DEFAULT .*$/, "") }.join(", ")
+
+                FuncDef.new(name, version, signature)
+              end
+            end
+        end
       end
 
       def self.next_migration_number(dir)

data/lib/generators/logidze/install/templates/hstore.rb.erb

@@ -1,4 +1,4 @@
-class <%= @migration_class_name %> < ActiveRecord::Migration<%= ActiveRecord::VERSION::MAJOR < 5 ? '' : '[5.0]' %>
+class <%= @migration_class_name %> < ActiveRecord::Migration[5.0]
   def change
     enable_extension :hstore
   end

data/lib/generators/logidze/install/templates/migration.rb.erb

@@ -1,246 +1,33 @@
-class <%= @migration_class_name %> < ActiveRecord::Migration<%= ActiveRecord::VERSION::MAJOR < 5 ? '' : '[5.0]' %>
-  require 'logidze/migration'
-  include Logidze::Migration
-
+class <%= @migration_class_name %> < ActiveRecord::Migration[5.0]
   def up
-    unless current_setting_missing_supported?
-      execute <<-SQL
-        DO $$
-          BEGIN
-          EXECUTE 'ALTER DATABASE ' || quote_ident(current_database()) || ' SET logidze.disabled=' || quote_literal('');
-          EXECUTE 'ALTER DATABASE ' || quote_ident(current_database()) || ' SET logidze.meta=' || quote_literal('');
-          END;
-        $$
-        LANGUAGE plpgsql;
-      SQL
-    end
-
-    <% if update? %>
-    execute <<-SQL
+<%- if update? -%>
+    # Drop legacy functions (<1.0)
+    execute <<~SQL
       DROP FUNCTION IF EXISTS logidze_version(bigint, jsonb);
       DROP FUNCTION IF EXISTS logidze_snapshot(jsonb);
       DROP FUNCTION IF EXISTS logidze_version(bigint, jsonb, text[]);
       DROP FUNCTION IF EXISTS logidze_snapshot(jsonb, text[]);
+      DROP FUNCTION IF EXISTS logidze_version(bigint, jsonb, timestamp with time zone, text[]);
+      DROP FUNCTION IF EXISTS logidze_snapshot(jsonb, text, text[]);
+      DROP FUNCTION IF EXISTS logidze_exclude_keys(jsonb, VARIADIC text[]);
+      DROP FUNCTION IF EXISTS logidze_compact_history(jsonb);
     SQL
-    <% end %>
-
-    execute <<-SQL
-      CREATE OR REPLACE FUNCTION logidze_version(v bigint, data jsonb, ts timestamp with time zone, blacklist text[] DEFAULT '{}') RETURNS jsonb AS $body$
-        DECLARE
-          buf jsonb;
-        BEGIN
-          buf := jsonb_build_object(
-                   'ts',
-                   (extract(epoch from ts) * 1000)::bigint,
-                   'v',
-                    v,
-                    'c',
-                    logidze_exclude_keys(data, VARIADIC array_append(blacklist, 'log_data'))
-                   );
-          IF coalesce(#{current_setting('logidze.meta')}, '') <> '' THEN
-            buf := jsonb_set(buf, ARRAY['m'], current_setting('logidze.meta')::jsonb);
-          END IF;
-          RETURN buf;
-        END;
-      $body$
-      LANGUAGE plpgsql;
-
-      CREATE OR REPLACE FUNCTION logidze_snapshot(item jsonb, ts_column text, blacklist text[] DEFAULT '{}') RETURNS jsonb AS $body$
-        DECLARE
-          ts timestamp with time zone;
-        BEGIN
-          IF ts_column IS NULL THEN
-            ts := statement_timestamp();
-          ELSE
-            ts := coalesce((item->>ts_column)::timestamp with time zone, statement_timestamp());
-          END IF;
-          return json_build_object(
-            'v', 1,
-            'h', jsonb_build_array(
-                   logidze_version(1, item, ts, blacklist)
-                 )
-            );
-        END;
-      $body$
-      LANGUAGE plpgsql;
-
-      CREATE OR REPLACE FUNCTION logidze_exclude_keys(obj jsonb, VARIADIC keys text[]) RETURNS jsonb AS $body$
-        DECLARE
-          res jsonb;
-          key text;
-        BEGIN
-          res := obj;
-          FOREACH key IN ARRAY keys
-          LOOP
-            res := res - key;
-          END LOOP;
-          RETURN res;
-        END;
-      $body$
-      LANGUAGE plpgsql;
-
-      CREATE OR REPLACE FUNCTION logidze_compact_history(log_data jsonb) RETURNS jsonb AS $body$
-        DECLARE
-          merged jsonb;
-        BEGIN
-          merged := jsonb_build_object(
-            'ts',
-            log_data#>'{h,1,ts}',
-            'v',
-            log_data#>'{h,1,v}',
-            'c',
-            (log_data#>'{h,0,c}') || (log_data#>'{h,1,c}')
-          );
-
-          IF (log_data#>'{h,1}' ? 'm') THEN
-            merged := jsonb_set(merged, ARRAY['m'], log_data#>'{h,1,m}');
-          END IF;
-
-          return jsonb_set(
-            log_data,
-            '{h}',
-            jsonb_set(
-              log_data->'h',
-              '{1}',
-              merged
-            ) - 0
-          );
-        END;
-      $body$
-      LANGUAGE plpgsql;
-
-      CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
-        DECLARE
-          changes jsonb;
-          version jsonb;
-          snapshot jsonb;
-          new_v integer;
-          size integer;
-          history_limit integer;
-          debounce_time integer;
-          current_version integer;
-          merged jsonb;
-          iterator integer;
-          item record;
-          columns_blacklist text[];
-          ts timestamp with time zone;
-          ts_column text;
-        BEGIN
-          ts_column := NULLIF(TG_ARGV[1], 'null');
-          columns_blacklist := COALESCE(NULLIF(TG_ARGV[2], 'null'), '{}');
-
-          IF TG_OP = 'INSERT' THEN
-            snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column, columns_blacklist);
-
-            IF snapshot#>>'{h, -1, c}' != '{}' THEN
-              NEW.log_data := snapshot;
-            END IF;
-
-          ELSIF TG_OP = 'UPDATE' THEN
-
-            IF OLD.log_data is NULL OR OLD.log_data = '{}'::jsonb THEN
-              snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column, columns_blacklist);
-              IF snapshot#>>'{h, -1, c}' != '{}' THEN
-                NEW.log_data := snapshot;
-              END IF;
-              RETURN NEW;
-            END IF;
-
-            history_limit := NULLIF(TG_ARGV[0], 'null');
-            debounce_time := NULLIF(TG_ARGV[3], 'null');
-
-            current_version := (NEW.log_data->>'v')::int;
-
-            IF ts_column IS NULL THEN
-              ts := statement_timestamp();
-            ELSE
-              ts := (to_jsonb(NEW.*)->>ts_column)::timestamp with time zone;
-              IF ts IS NULL OR ts = (to_jsonb(OLD.*)->>ts_column)::timestamp with time zone THEN
-                ts := statement_timestamp();
-              END IF;
-            END IF;
-
-            IF NEW = OLD THEN
-              RETURN NEW;
-            END IF;
-
-            IF current_version < (NEW.log_data#>>'{h,-1,v}')::int THEN
-              iterator := 0;
-              FOR item in SELECT * FROM jsonb_array_elements(NEW.log_data->'h')
-              LOOP
-                IF (item.value->>'v')::int > current_version THEN
-                  NEW.log_data := jsonb_set(
-                    NEW.log_data,
-                    '{h}',
-                    (NEW.log_data->'h') - iterator
-                  );
-                END IF;
-                iterator := iterator + 1;
-              END LOOP;
-            END IF;
-
-            changes := hstore_to_jsonb_loose(
-              hstore(NEW.*) - hstore(OLD.*)
-            );
-
-            new_v := (NEW.log_data#>>'{h,-1,v}')::int + 1;
-
-            size := jsonb_array_length(NEW.log_data->'h');
-            version := logidze_version(new_v, changes, ts, columns_blacklist);
-
-            IF version->>'c' = '{}' THEN
-              RETURN NEW;
-            END IF;
-
-            IF (
-              debounce_time IS NOT NULL AND
-              (version->>'ts')::bigint - (NEW.log_data#>'{h,-1,ts}')::text::bigint <= debounce_time
-            ) THEN
-              -- merge new version with the previous one
-              new_v := (NEW.log_data#>>'{h,-1,v}')::int;
-              version := logidze_version(new_v, (NEW.log_data#>'{h,-1,c}')::jsonb || changes, ts, columns_blacklist);
-              -- remove the previous version from log
-              NEW.log_data := jsonb_set(
-                NEW.log_data,
-                '{h}',
-                (NEW.log_data->'h') - (size - 1)
-              );
-            END IF;
-
-            NEW.log_data := jsonb_set(
-              NEW.log_data,
-              ARRAY['h', size::text],
-              version,
-              true
-            );
-
-            NEW.log_data := jsonb_set(
-              NEW.log_data,
-              '{v}',
-              to_jsonb(new_v)
-            );
-
-            IF history_limit IS NOT NULL AND history_limit = size THEN
-              NEW.log_data := logidze_compact_history(NEW.log_data);
-            END IF;
-          END IF;
 
-          return NEW;
-        END;
-        $body$
-        LANGUAGE plpgsql;
+<%- end -%>
+    execute <<~SQL
+<%- function_definitions.each do |f| -%>
+<%= inject_sql("#{f.name}.sql", indent: 6) %>
+<%- end -%>
     SQL
   end
 
   def down
-    <% unless update? %>
-    execute <<-SQL
-      DROP FUNCTION logidze_version(bigint, jsonb, timestamp with time zone, text[]) CASCADE;
-      DROP FUNCTION logidze_exclude_keys(jsonb, text[]) CASCADE;
-      DROP FUNCTION logidze_compact_history(jsonb) CASCADE;
-      DROP FUNCTION logidze_snapshot(jsonb, text, text[]) CASCADE;
-      DROP FUNCTION logidze_logger() CASCADE;
+<%- unless update? -%>
+    execute <<~SQL
+  <%- function_definitions.each do |f| -%>
+      DROP FUNCTION IF EXISTS <%= f.name %>(<%= f.signature %>) CASCADE;
+  <%- end -%>
     SQL
-    <% end %>
+<%- end -%>
   end
 end