logidze 0.11.0 → 1.2.0

data/lib/generators/logidze/fx_helper.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Logidze
+  module Generators
+    # Adds --fx option and provide #fx? method
+    module FxHelper
+      def self.included(base)
+        base.class_option :fx, type: :boolean, optional: true,
+                               desc: "Define whether to use fx gem functionality"
+      end
+
+      def fx?
+        options[:fx] || (options[:fx] != false && defined?(::Fx::SchemaDumper))
+      end
+    end
+  end
+end

data/lib/generators/logidze/inject_sql.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Logidze
+  module Generators
+    module InjectSql
+      def inject_sql(source, indent: 4)
+        source = ::File.expand_path(find_in_source_paths(source.to_s))
+
+        indent(
+          ERB.new(::File.binread(source)).tap do |erb|
+            erb.filename = source
+          end.result(instance_eval("binding")), # rubocop:disable Style/EvalWithLocation
+          indent
+        )
+      end
+    end
+  end
+end

data/lib/generators/logidze/install/USAGE

@@ -1,7 +1,12 @@
 Description:
   Generates the necessary files to get you up and running with Logidze gem
-  
+
 Examples:
   rails generate logidze:install
 
     This will generate the core migration file with trigger function defined.
+
+  rails generate logidze:install --fx
+
+    This will generate schema.rb compatible migration with `create_function` definitions and separate SQL files.
+    The fx gem must be installed.

data/lib/generators/logidze/install/functions/logidze_capture_exception.sql

@@ -0,0 +1,23 @@
+CREATE OR REPLACE FUNCTION logidze_capture_exception(error_data jsonb) RETURNS boolean AS $body$
+  -- version: 1
+BEGIN
+  -- Feel free to change this function to change Logidze behavior on exception.
+  --
+  -- Return `false` to raise exception or `true` to commit record changes.
+  --
+  -- `error_data` contains:
+  --   - returned_sqlstate
+  --   - message_text
+  --   - pg_exception_detail
+  --   - pg_exception_hint
+  --   - pg_exception_context
+  --   - schema_name
+  --   - table_name
+  -- Learn more about available keys:
+  -- https://www.postgresql.org/docs/9.6/plpgsql-control-structures.html#PLPGSQL-EXCEPTION-DIAGNOSTICS-VALUES
+  --
+
+  return false;
+END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_compact_history.sql

@@ -0,0 +1,38 @@
+CREATE OR REPLACE FUNCTION logidze_compact_history(log_data jsonb, cutoff integer DEFAULT 1) RETURNS jsonb AS $body$
+  -- version: 1
+  DECLARE
+    merged jsonb;
+  BEGIN
+    LOOP
+      merged := jsonb_build_object(
+        'ts',
+        log_data#>'{h,1,ts}',
+        'v',
+        log_data#>'{h,1,v}',
+        'c',
+        (log_data#>'{h,0,c}') || (log_data#>'{h,1,c}')
+      );
+
+      IF (log_data#>'{h,1}' ? 'm') THEN
+        merged := jsonb_set(merged, ARRAY['m'], log_data#>'{h,1,m}');
+      END IF;
+
+      log_data := jsonb_set(
+        log_data,
+        '{h}',
+        jsonb_set(
+          log_data->'h',
+          '{1}',
+          merged
+        ) - 0
+      );
+
+      cutoff := cutoff - 1;
+
+      EXIT WHEN cutoff <= 0;
+    END LOOP;
+
+    return log_data;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_filter_keys.sql

@@ -0,0 +1,27 @@
+CREATE OR REPLACE FUNCTION logidze_filter_keys(obj jsonb, keys text[], include_columns boolean DEFAULT false) RETURNS jsonb AS $body$
+  -- version: 1
+  DECLARE
+    res jsonb;
+    key text;
+  BEGIN
+    res := '{}';
+
+    IF include_columns THEN
+      FOREACH key IN ARRAY keys
+      LOOP
+        IF obj ? key THEN
+          res = jsonb_insert(res, ARRAY[key], obj->key);
+        END IF;
+      END LOOP;
+    ELSE
+      res = obj;
+      FOREACH key IN ARRAY keys
+      LOOP
+        res = res - key;
+      END LOOP;
+    END IF;
+
+    RETURN res;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_logger.sql

@@ -0,0 +1,203 @@
+CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
+  -- version: 2
+  DECLARE
+    changes jsonb;
+    version jsonb;
+    snapshot jsonb;
+    new_v integer;
+    size integer;
+    history_limit integer;
+    debounce_time integer;
+    current_version integer;
+    k text;
+    iterator integer;
+    item record;
+    columns text[];
+    include_columns boolean;
+    ts timestamp with time zone;
+    ts_column text;
+    err_sqlstate text;
+    err_message text;
+    err_detail text;
+    err_hint text;
+    err_context text;
+    err_table_name text;
+    err_schema_name text;
+    err_jsonb jsonb;
+    err_captured boolean;
+  BEGIN
+    ts_column := NULLIF(TG_ARGV[1], 'null');
+    columns := NULLIF(TG_ARGV[2], 'null');
+    include_columns := NULLIF(TG_ARGV[3], 'null');
+
+    IF TG_OP = 'INSERT' THEN
+      IF columns IS NOT NULL THEN
+        snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column, columns, include_columns);
+      ELSE
+        snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column);
+      END IF;
+
+      IF snapshot#>>'{h, -1, c}' != '{}' THEN
+        NEW.log_data := snapshot;
+      END IF;
+
+    ELSIF TG_OP = 'UPDATE' THEN
+
+      IF OLD.log_data is NULL OR OLD.log_data = '{}'::jsonb THEN
+        IF columns IS NOT NULL THEN
+          snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column, columns, include_columns);
+        ELSE
+          snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column);
+        END IF;
+
+        IF snapshot#>>'{h, -1, c}' != '{}' THEN
+          NEW.log_data := snapshot;
+        END IF;
+        RETURN NEW;
+      END IF;
+
+      history_limit := NULLIF(TG_ARGV[0], 'null');
+      debounce_time := NULLIF(TG_ARGV[4], 'null');
+
+      current_version := (NEW.log_data->>'v')::int;
+
+      IF ts_column IS NULL THEN
+        ts := statement_timestamp();
+      ELSE
+        ts := (to_jsonb(NEW.*)->>ts_column)::timestamp with time zone;
+        IF ts IS NULL OR ts = (to_jsonb(OLD.*)->>ts_column)::timestamp with time zone THEN
+          ts := statement_timestamp();
+        END IF;
+      END IF;
+
+      IF NEW = OLD THEN
+        RETURN NEW;
+      END IF;
+
+      IF current_version < (NEW.log_data#>>'{h,-1,v}')::int THEN
+        iterator := 0;
+        FOR item in SELECT * FROM jsonb_array_elements(NEW.log_data->'h')
+        LOOP
+          IF (item.value->>'v')::int > current_version THEN
+            NEW.log_data := jsonb_set(
+              NEW.log_data,
+              '{h}',
+              (NEW.log_data->'h') - iterator
+            );
+          END IF;
+          iterator := iterator + 1;
+        END LOOP;
+      END IF;
+
+      changes := '{}';
+
+      IF (coalesce(current_setting('logidze.full_snapshot', true), '') = 'on') THEN
+        BEGIN
+          changes = hstore_to_jsonb_loose(hstore(NEW.*));
+        EXCEPTION
+          WHEN NUMERIC_VALUE_OUT_OF_RANGE THEN
+            changes = row_to_json(NEW.*)::jsonb;
+            FOR k IN (SELECT key FROM jsonb_each(changes))
+            LOOP
+              IF jsonb_typeof(changes->k) = 'object' THEN
+                changes = jsonb_set(changes, ARRAY[k], to_jsonb(changes->>k));
+              END IF;
+            END LOOP;
+        END;
+      ELSE
+        BEGIN
+          changes = hstore_to_jsonb_loose(
+                hstore(NEW.*) - hstore(OLD.*)
+            );
+        EXCEPTION
+          WHEN NUMERIC_VALUE_OUT_OF_RANGE THEN
+            changes = (SELECT
+              COALESCE(json_object_agg(key, value), '{}')::jsonb
+              FROM
+              jsonb_each(row_to_json(NEW.*)::jsonb)
+              WHERE NOT jsonb_build_object(key, value) <@ row_to_json(OLD.*)::jsonb);
+            FOR k IN (SELECT key FROM jsonb_each(changes))
+            LOOP
+              IF jsonb_typeof(changes->k) = 'object' THEN
+                changes = jsonb_set(changes, ARRAY[k], to_jsonb(changes->>k));
+              END IF;
+            END LOOP;
+        END;
+      END IF;
+
+      changes = changes - 'log_data';
+
+      IF columns IS NOT NULL THEN
+        changes = logidze_filter_keys(changes, columns, include_columns);
+      END IF;
+
+      IF changes = '{}' THEN
+        RETURN NEW;
+      END IF;
+
+      new_v := (NEW.log_data#>>'{h,-1,v}')::int + 1;
+
+      size := jsonb_array_length(NEW.log_data->'h');
+      version := logidze_version(new_v, changes, ts);
+
+      IF (
+        debounce_time IS NOT NULL AND
+        (version->>'ts')::bigint - (NEW.log_data#>'{h,-1,ts}')::text::bigint <= debounce_time
+      ) THEN
+        -- merge new version with the previous one
+        new_v := (NEW.log_data#>>'{h,-1,v}')::int;
+        version := logidze_version(new_v, (NEW.log_data#>'{h,-1,c}')::jsonb || changes, ts);
+        -- remove the previous version from log
+        NEW.log_data := jsonb_set(
+          NEW.log_data,
+          '{h}',
+          (NEW.log_data->'h') - (size - 1)
+        );
+      END IF;
+
+      NEW.log_data := jsonb_set(
+        NEW.log_data,
+        ARRAY['h', size::text],
+        version,
+        true
+      );
+
+      NEW.log_data := jsonb_set(
+        NEW.log_data,
+        '{v}',
+        to_jsonb(new_v)
+      );
+
+      IF history_limit IS NOT NULL AND history_limit <= size THEN
+        NEW.log_data := logidze_compact_history(NEW.log_data, size - history_limit + 1);
+      END IF;
+    END IF;
+
+    return NEW;
+  EXCEPTION
+    WHEN OTHERS THEN
+      GET STACKED DIAGNOSTICS err_sqlstate = RETURNED_SQLSTATE,
+                              err_message = MESSAGE_TEXT,
+                              err_detail = PG_EXCEPTION_DETAIL,
+                              err_hint = PG_EXCEPTION_HINT,
+                              err_context = PG_EXCEPTION_CONTEXT,
+                              err_schema_name = SCHEMA_NAME,
+                              err_table_name = TABLE_NAME;
+      err_jsonb := jsonb_build_object(
+        'returned_sqlstate', err_sqlstate,
+        'message_text', err_message,
+        'pg_exception_detail', err_detail,
+        'pg_exception_hint', err_hint,
+        'pg_exception_context', err_context,
+        'schema_name', err_schema_name,
+        'table_name', err_table_name
+      );
+      err_captured = logidze_capture_exception(err_jsonb);
+      IF err_captured THEN
+        return NEW;
+      ELSE
+        RAISE;
+      END IF;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_snapshot.sql

@@ -0,0 +1,33 @@
+CREATE OR REPLACE FUNCTION logidze_snapshot(item jsonb, ts_column text DEFAULT NULL, columns text[] DEFAULT NULL, include_columns boolean DEFAULT false) RETURNS jsonb AS $body$
+  -- version: 3
+  DECLARE
+    ts timestamp with time zone;
+    k text;
+  BEGIN
+    item = item - 'log_data';
+    IF ts_column IS NULL THEN
+      ts := statement_timestamp();
+    ELSE
+      ts := coalesce((item->>ts_column)::timestamp with time zone, statement_timestamp());
+    END IF;
+
+    IF columns IS NOT NULL THEN
+      item := logidze_filter_keys(item, columns, include_columns);
+    END IF;
+
+    FOR k IN (SELECT key FROM jsonb_each(item))
+    LOOP
+      IF jsonb_typeof(item->k) = 'object' THEN
+         item := jsonb_set(item, ARRAY[k], to_jsonb(item->>k));
+      END IF;
+    END LOOP;
+
+    return json_build_object(
+      'v', 1,
+      'h', jsonb_build_array(
+              logidze_version(1, item, ts)
+            )
+      );
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_version.sql

@@ -0,0 +1,21 @@
+CREATE OR REPLACE FUNCTION logidze_version(v bigint, data jsonb, ts timestamp with time zone) RETURNS jsonb AS $body$
+  -- version: 2
+  DECLARE
+    buf jsonb;
+  BEGIN
+    data = data - 'log_data';
+    buf := jsonb_build_object(
+              'ts',
+              (extract(epoch from ts) * 1000)::bigint,
+              'v',
+              v,
+              'c',
+              data
+              );
+    IF coalesce(current_setting('logidze.meta', true), '') <> '' THEN
+      buf := jsonb_insert(buf, '{m}', current_setting('logidze.meta')::jsonb);
+    END IF;
+    RETURN buf;
+  END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/install_generator.rb

@@ -2,19 +2,28 @@
 
 require "rails/generators"
 require "rails/generators/active_record"
+require "logidze/utils/function_definitions"
+require_relative "../inject_sql"
+require_relative "../fx_helper"
+
+using RubyNext
 
 module Logidze
   module Generators
     class InstallGenerator < ::Rails::Generators::Base # :nodoc:
       include Rails::Generators::Migration
+      include InjectSql
+      include FxHelper
 
       source_root File.expand_path("templates", __dir__)
+      source_paths << File.expand_path("functions", __dir__)
 
       class_option :update, type: :boolean, optional: true,
                             desc: "Define whether this is an update migration"
 
       def generate_migration
-        migration_template "migration.rb.erb", "db/migrate/#{migration_name}.rb"
+        migration_template = fx? ? "migration_fx.rb.erb" : "migration.rb.erb"
+        migration_template migration_template, "db/migrate/#{migration_name}.rb"
       end
 
       def generate_hstore_migration
@@ -23,6 +32,16 @@ module Logidze
         migration_template "hstore.rb.erb", "db/migrate/enable_hstore.rb"
       end
 
+      def generate_fx_functions
+        return unless fx?
+
+        function_definitions.each do |fdef|
+          next if fdef.version == previous_version_for(fdef.name)
+
+          template "#{fdef.name}.sql", "db/functions/#{fdef.name}_v#{fdef.version.to_s.rjust(2, "0")}.sql"
+        end
+      end
+
       no_tasks do
         def migration_name
           if update?
@@ -39,6 +58,29 @@ module Logidze
         def update?
           options[:update]
         end
+
+        def previous_version_for(name)
+          all_functions.filter_map { |path| Regexp.last_match[1].to_i if path =~ %r{#{name}_v(\d+).sql} }.max
+        end
+
+        def all_functions
+          @all_functions ||=
+            begin
+              res = nil
+              in_root do
+                res = if File.directory?("db/functions")
+                  Dir.entries("db/functions")
+                else
+                  []
+                end
+              end
+              res
+            end
+        end
+
+        def function_definitions
+          @function_definitions ||= Logidze::Utils::FunctionDefinitions.from_fs
+        end
       end
 
       def self.next_migration_number(dir)