logidze 1.1.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6c441d5e6e60ef695eb354e4bfb5be9fa182bd8b70755d829aee6eb2781e401
-  data.tar.gz: d64374a0d6327f2f713fd7341c31de93472f32d045cb5c825c35874c756307d4
+  metadata.gz: 68d689e98373be99d5039ebc70fc05d369a75b94e20ef0bb9d0d9d43e70c3887
+  data.tar.gz: 3c3c24bd36f3af269d1443562f271ede71d222dfea5ad8d52c9199ca72d91244
 SHA512:
-  metadata.gz: f3f453e410de263ed8b0704aece35e6274dfc19c9074a8f75450e84033318b149ae81c9e1f48cc9d515e523b95c98c14cbd3453a7c41db1690febfe8060f640d
-  data.tar.gz: dbf4a22b357889bc0d2aaeda52bffc8bc245cdc12d2cb34d0f7c9ccec7c0f60807ecc3cbd5272fee567420024309f8be05f6e1e8c9f45334ab3d2ad155a97a12
+  metadata.gz: 78217c05e5bab5aa2ce2a865c1a9e84cdf76ceb56e52e951394c58e098f295732981fc911a499d6c6fd2fa81f3ea25636dcd1708f8940df068dfc37ca367cfac
+  data.tar.gz: 0e050d100d842aaeb75b9f7764d4ee5e862ace9cfe3a880f0538839898f87ebf317884d51136be8600e374f8ceeed38605aa44b0952b16f270df20b5017e99c2

data/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 ## master (unreleased)
 
+## 1.2.2 (2022-07-13)
+
+- [Fixes [#209](https://github.com/palkan/logidze/issues/209)] Fix tracking JSONB column changes. ([@baygeldin][])
+
+## 1.2.1 (2022-01-13)
+
+- [Fixes [#207](https://github.com/palkan/logidze/issues/207)] Add support for the use of `table_name_prefix` or `table_name_suffix`. ([@cavi21][])
+
+- [Fixes [#205](https://github.com/palkan/logidze/issues/205)] Allow `rails destroy logidze:model SomeModel` to delete the migration file. ([@danielmklein][])
+
+## 1.2.0 (2021-06-11)
+
+- Add user-defined exception handling ([@skryukov][])
+
+By default, Logidze raises an exception which causes the entire transaction to fail.
+To change this behavior, it's now possible to override `logidze_capture_exception(error_data jsonb)` function.
+
+- [Fixes [#69](https://github.com/palkan/logidze/issues/69)] Fallback on NUMERIC_VALUE_OUT_OF_RANGE exception ([@skryukov][])
+
+- [Fixes [#192](https://github.com/palkan/logidze/issues/192)] Skip `log_data` column during `apply_column_diff` ([@skryukov][])
+
 ## 1.1.0 (2021-03-31)
 
 - Add pending upgrade checks [Experimental]. ([@skryukov][])
@@ -347,3 +368,7 @@ This is a quick fix for a more general problem (see [#59](https://github.com/pal
 [@duderman]: https://github.com/duderman
 [@oleg-kiviljov]: https://github.com/oleg-kiviljov
 [@skryukov]: https://github.com/skryukov
+[@bf4]: https://github.com/bf4
+[@cavi21]: https://github.com/cavi21
+[@danielmklein]: https://github.com/danielmklein
+[@baygeldin]: https://github.com/baygeldin

data/README.md

@@ -34,6 +34,7 @@ Other requirements:
   - [Log size limits](#log-size-limits)
   - [Tracking only selected columns](#tracking-only-selected-columns)
   - [Logs timestamps](#logs-timestamps)
+  - [Undoing a Generated Invocation](#undoing-a-generated-invocation)
 - [Usage](#usage)
   - [Basic API](#basic-api)
   - [Track meta information](#track-meta-information)
@@ -44,6 +45,7 @@ Other requirements:
   - [Associations versioning](#associations-versioning)
 - [Dealing with large logs](#dealing-with-large-logs)
 - [Handling records deletion](#handling-records-deletion)
+- [Handling PG exceptions](#handling-pg-exceptions)
 - [Upgrading](#upgrading)
 - [Log format](#log-format)
 - [Troubleshooting 🚨](#troubleshooting)
@@ -54,7 +56,7 @@ Other requirements:
 Add Logidze to your application's Gemfile:
 
 ```ruby
-gem "logidze", "~> 1.0.0"
+gem "logidze", "~> 1.1"
 ```
 
 Install required DB extensions and create trigger function:
@@ -133,8 +135,8 @@ Model.create_logidze_snapshot
 Model.create_logidze_snapshot(timestamp: :created_at)
 
 # filter columns
-Model.create_logidze_snapshot(only: %(name))
-Model.create_logidze_snapshot(except: %(password))
+Model.create_logidze_snapshot(only: %w[name])
+Model.create_logidze_snapshot(except: %w[password])
 
 # or call a similar method (but with !) on a record
 
@@ -177,6 +179,16 @@ bundle exec rails generate logidze:model Post --timestamp_column time
 bundle exec rails generate logidze:model Post --timestamp_column nil # "null" and "false" will also work
 ```
 
+### Undoing a Generated Invocation
+
+If you would like to re-do your `rails generate` anew, as with other generators you can use `rails destroy` to revert it, which will delete the migration file and undo the injection of `has_logidze` into the model file:
+
+```sh
+bundle exec rails destroy logidze:model Post
+```
+
+**IMPORTANT**: If you use non-UTC time zone for Active Record (`config.active_record.default_timezone`), you MUST always infer log timestamps from a timestamp column (e.g., when back-filling data); otherwise, you may end up with inconsistent logs ([#199](https://github.com/palkan/logidze/issues/199)). In general, we recommend using UTC as the database time unless there is a very strong reason not to.
+
 ## Usage
 
 ### Basic API
@@ -273,11 +285,13 @@ Logidze.append_on_undo = true
 You can store any meta information you want inside your version (it could be IP address, user agent, etc.). To add it you should wrap your code with a block:
 
 ```ruby
-Logidze.with_meta(ip: request.ip) do
+Logidze.with_meta({ip: request.ip}) do
   post.save!
 end
 ```
 
+**NOTE:** You should pass metadata as a Hash; passing keyword arguments doesn't work in Ruby 3.0+.
+
 Meta expects a hash to be passed so you won't need to encode and decode JSON manually.
 
 By default `.with_meta` wraps the block into a DB transaction. That could lead to an unexpected behavior, especially, when using `.with_meta` within an around_action. To avoid wrapping the block into a DB transaction use `transactional: false` option.
@@ -434,6 +448,15 @@ If you want to keep changes history after records deletion as well, consider usi
 
 See also the discussion: [#61](https://github.com/palkan/logidze/issues/61).
 
+## Handling PG exceptions
+
+By default, Logidze raises an exception which causes the entire transaction to fail.
+To change this behavior, it's now possible to override `logidze_capture_exception(error_data jsonb)` function.
+
+For example, you may want to raise a warning instead of an exception and complete the transaction without updating log_data.
+
+Related issues: [#193](https://github.com/palkan/logidze/issues/193)
+
 ## Upgrading
 
 We try to make an upgrade process as simple as possible. For now, the only required action is to create and run a migration:
@@ -462,7 +485,7 @@ $ bundle exec rails generate logidze:model Post --update --only=title,body,ratin
 
 ### Pending upgrade check [Experimental]
 
-Logidze can check for a pending upgrade. Use `Logidze.pending_upgrade = :warn` to be notified by warning, or `Logidze.pending_upgrade = :error` if you want Logidze to raise an error.
+Logidze can check for a pending upgrade. Use `Logidze.on_pending_upgrade = :warn` to be notified by warning, or `Logidze.on_pending_upgrade = :error` if you want Logidze to raise an error.
 
 ### Upgrading from 0.x to 1.0 (edge)
 
@@ -544,11 +567,27 @@ First, when restoring data dumps you should consider using `--disable-triggers`
 
 When restoring data dumps for a particular PostgreSQL schema (e.g., when using Apartment), you may encounter the issue with non-existent Logidze functions. That happens because `pg_dump` adds `SELECT pg_catalog.set_config('search_path', '', false);`, and, thus, breaks our existing triggers/functions, because they live either in "public" or in a tenant's namespace (see [this thread](https://postgrespro.com/list/thread-id/2448092)).
 
+### `PG::NumericValueOutOfRange: ERROR: value overflows numeric format`
+
+Due to the usage of `hstore_to_jsonb_loose` under the hood, there could be a situation when you have a string representing a number in the scientific notation (e.g., "557236406134e62000323100"). Postgres would try to convert it to a number (a pretty big one, for sure) and fail with the exception.
+
+Related issues: [#69](https://github.com/palkan/logidze/issues/69).
+
 ## Development
 
-We use [Dip](https://github.com/bibendi/dip) for development. Provision the project by running `dip provision` and then use `dip bundle`, `dip rspec` or `dip bash` to interact with a Docker development environment.
+This project requires a PostgreSQL instance running with the following setup:
+
+```sh
+# For testing
+createdb -h postgres -U postgres logidze_test
+
+# For benchmarks
+createdb -h postgres -U postgres logidze_bench
+createdb -h postgres -U postgres logidze_perf_bench
+psql -d logidze_bench -c 'CREATE EXTENSION IF NOT EXISTS hstore;'
+```
 
-If you prefer developing on your local machine, make user you have Postgres installed and run `./bin/setup`.
+This project is compatible with [Reusable Docker environment](https://evilmartians.com/chronicles/reusable-development-containers-with-docker-compose-and-dip) setup.
 
 ## Contributing
 

data/lib/generators/logidze/install/functions/logidze_capture_exception.sql

@@ -0,0 +1,23 @@
+CREATE OR REPLACE FUNCTION logidze_capture_exception(error_data jsonb) RETURNS boolean AS $body$
+  -- version: 1
+BEGIN
+  -- Feel free to change this function to change Logidze behavior on exception.
+  --
+  -- Return `false` to raise exception or `true` to commit record changes.
+  --
+  -- `error_data` contains:
+  --   - returned_sqlstate
+  --   - message_text
+  --   - pg_exception_detail
+  --   - pg_exception_hint
+  --   - pg_exception_context
+  --   - schema_name
+  --   - table_name
+  -- Learn more about available keys:
+  -- https://www.postgresql.org/docs/9.6/plpgsql-control-structures.html#PLPGSQL-EXCEPTION-DIAGNOSTICS-VALUES
+  --
+
+  return false;
+END;
+$body$
+LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_logger.sql

@@ -1,5 +1,5 @@
 CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
-  -- version: 1
+  -- version: 3
   DECLARE
     changes jsonb;
     version jsonb;
@@ -9,26 +9,32 @@ CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
     history_limit integer;
     debounce_time integer;
     current_version integer;
-    merged jsonb;
+    k text;
     iterator integer;
     item record;
     columns text[];
     include_columns boolean;
     ts timestamp with time zone;
     ts_column text;
+    err_sqlstate text;
+    err_message text;
+    err_detail text;
+    err_hint text;
+    err_context text;
+    err_table_name text;
+    err_schema_name text;
+    err_jsonb jsonb;
+    err_captured boolean;
   BEGIN
     ts_column := NULLIF(TG_ARGV[1], 'null');
     columns := NULLIF(TG_ARGV[2], 'null');
     include_columns := NULLIF(TG_ARGV[3], 'null');
 
     IF TG_OP = 'INSERT' THEN
-      -- always exclude log_data column
-      changes := to_jsonb(NEW.*) - 'log_data';
-
       IF columns IS NOT NULL THEN
-        snapshot = logidze_snapshot(changes, ts_column, columns, include_columns);
+        snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column, columns, include_columns);
       ELSE
-        snapshot = logidze_snapshot(changes, ts_column);
+        snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column);
       END IF;
 
       IF snapshot#>>'{h, -1, c}' != '{}' THEN
@@ -38,13 +44,10 @@ CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
     ELSIF TG_OP = 'UPDATE' THEN
 
       IF OLD.log_data is NULL OR OLD.log_data = '{}'::jsonb THEN
-        -- always exclude log_data column
-        changes := to_jsonb(NEW.*) - 'log_data';
-
         IF columns IS NOT NULL THEN
-          snapshot = logidze_snapshot(changes, ts_column, columns, include_columns);
+          snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column, columns, include_columns);
         ELSE
-          snapshot = logidze_snapshot(changes, ts_column);
+          snapshot = logidze_snapshot(to_jsonb(NEW.*), ts_column);
         END IF;
 
         IF snapshot#>>'{h, -1, c}' != '{}' THEN
@@ -67,7 +70,7 @@ CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
         END IF;
       END IF;
 
-      IF NEW = OLD THEN
+      IF to_jsonb(NEW.*) = to_jsonb(OLD.*) THEN
         RETURN NEW;
       END IF;
 
@@ -89,11 +92,37 @@ CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
       changes := '{}';
 
       IF (coalesce(current_setting('logidze.full_snapshot', true), '') = 'on') THEN
-        changes = hstore_to_jsonb_loose(hstore(NEW.*));
+        BEGIN
+          changes = hstore_to_jsonb_loose(hstore(NEW.*));
+        EXCEPTION
+          WHEN NUMERIC_VALUE_OUT_OF_RANGE THEN
+            changes = row_to_json(NEW.*)::jsonb;
+            FOR k IN (SELECT key FROM jsonb_each(changes))
+            LOOP
+              IF jsonb_typeof(changes->k) = 'object' THEN
+                changes = jsonb_set(changes, ARRAY[k], to_jsonb(changes->>k));
+              END IF;
+            END LOOP;
+        END;
       ELSE
-        changes = hstore_to_jsonb_loose(
-          hstore(NEW.*) - hstore(OLD.*)
-        );
+        BEGIN
+          changes = hstore_to_jsonb_loose(
+                hstore(NEW.*) - hstore(OLD.*)
+            );
+        EXCEPTION
+          WHEN NUMERIC_VALUE_OUT_OF_RANGE THEN
+            changes = (SELECT
+              COALESCE(json_object_agg(key, value), '{}')::jsonb
+              FROM
+              jsonb_each(row_to_json(NEW.*)::jsonb)
+              WHERE NOT jsonb_build_object(key, value) <@ row_to_json(OLD.*)::jsonb);
+            FOR k IN (SELECT key FROM jsonb_each(changes))
+            LOOP
+              IF jsonb_typeof(changes->k) = 'object' THEN
+                changes = jsonb_set(changes, ARRAY[k], to_jsonb(changes->>k));
+              END IF;
+            END LOOP;
+        END;
       END IF;
 
       changes = changes - 'log_data';
@@ -145,6 +174,30 @@ CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
     END IF;
 
     return NEW;
+  EXCEPTION
+    WHEN OTHERS THEN
+      GET STACKED DIAGNOSTICS err_sqlstate = RETURNED_SQLSTATE,
+                              err_message = MESSAGE_TEXT,
+                              err_detail = PG_EXCEPTION_DETAIL,
+                              err_hint = PG_EXCEPTION_HINT,
+                              err_context = PG_EXCEPTION_CONTEXT,
+                              err_schema_name = SCHEMA_NAME,
+                              err_table_name = TABLE_NAME;
+      err_jsonb := jsonb_build_object(
+        'returned_sqlstate', err_sqlstate,
+        'message_text', err_message,
+        'pg_exception_detail', err_detail,
+        'pg_exception_hint', err_hint,
+        'pg_exception_context', err_context,
+        'schema_name', err_schema_name,
+        'table_name', err_table_name
+      );
+      err_captured = logidze_capture_exception(err_jsonb);
+      IF err_captured THEN
+        return NEW;
+      ELSE
+        RAISE;
+      END IF;
   END;
 $body$
 LANGUAGE plpgsql;

data/lib/generators/logidze/install/functions/logidze_snapshot.sql

@@ -1,9 +1,10 @@
 CREATE OR REPLACE FUNCTION logidze_snapshot(item jsonb, ts_column text DEFAULT NULL, columns text[] DEFAULT NULL, include_columns boolean DEFAULT false) RETURNS jsonb AS $body$
-  -- version: 2
+  -- version: 3
   DECLARE
     ts timestamp with time zone;
     k text;
   BEGIN
+    item = item - 'log_data';
     IF ts_column IS NULL THEN
       ts := statement_timestamp();
     ELSE

data/lib/generators/logidze/install/functions/logidze_version.sql

@@ -1,8 +1,9 @@
 CREATE OR REPLACE FUNCTION logidze_version(v bigint, data jsonb, ts timestamp with time zone) RETURNS jsonb AS $body$
-  -- version: 1
+  -- version: 2
   DECLARE
     buf jsonb;
   BEGIN
+    data = data - 'log_data';
     buf := jsonb_build_object(
               'ts',
               (extract(epoch from ts) * 1000)::bigint,

data/lib/generators/logidze/model/model_generator.rb

@@ -45,7 +45,7 @@ module Logidze
           warn "Use only one: --only or --except"
           exit(1)
         end
-        migration_template "migration.rb.erb", "db/migrate/#{migration_file_name}"
+        migration_template "migration.rb.erb", "db/migrate/#{migration_name}.rb"
       end
 
       def generate_fx_trigger
@@ -73,8 +73,9 @@ module Logidze
           end
         end
 
-        def migration_file_name
-          "#{migration_name}.rb"
+        def full_table_name
+          config = ActiveRecord::Base
+          "#{config.table_name_prefix}#{table_name}#{config.table_name_suffix}"
         end
 
         def limit

data/lib/generators/logidze/model/templates/migration.rb.erb

@@ -12,14 +12,18 @@ class <%= @migration_class_name %> < ActiveRecord::Migration[<%= ActiveRecord::M
       dir.up do
     <%- if update? -%>
         # Drop legacy trigger if any (<1.0)
-        execute "DROP TRIGGER IF EXISTS logidze_on_<%= table_name %> on <%= table_name %>;"
+        execute <<~SQL
+          DROP TRIGGER IF EXISTS "logidze_on_<%= full_table_name %>" on "<%= full_table_name %>";
+        SQL
 
     <%- end -%>
         create_trigger :logidze_on_<%= table_name %>, on: :<%= table_name %>
       end
 
       dir.down do
-        execute "DROP TRIGGER IF EXISTS logidze_on_<%= table_name %> on <%= table_name %>;"
+        execute <<~SQL
+          DROP TRIGGER IF EXISTS "logidze_on_<%= full_table_name %>" on "<%= full_table_name %>";
+        SQL
       end
     end
   <%- end -%>
@@ -27,7 +31,9 @@ class <%= @migration_class_name %> < ActiveRecord::Migration[<%= ActiveRecord::M
     reversible do |dir|
       dir.up do
   <%- if update? -%>
-        execute "DROP TRIGGER IF EXISTS logidze_on_<%= table_name %> on <%= table_name %>;"
+        execute <<~SQL
+          DROP TRIGGER IF EXISTS "logidze_on_<%= full_table_name %>" on "<%= full_table_name %>";
+        SQL
 
   <%- end -%>
         execute <<~SQL
@@ -44,7 +50,9 @@ class <%= @migration_class_name %> < ActiveRecord::Migration[<%= ActiveRecord::M
         # Uncomment this line if you want to raise an error.
         # raise ActiveRecord::IrreversibleMigration
   <%- else -%>
-        execute "DROP TRIGGER IF EXISTS logidze_on_<%= table_name %> on <%= table_name %>;"
+        execute <<~SQL
+          DROP TRIGGER IF EXISTS "logidze_on_<%= full_table_name %>" on "<%= full_table_name %>";
+        SQL
   <%- end -%>
       end
     end
@@ -54,7 +62,7 @@ class <%= @migration_class_name %> < ActiveRecord::Migration[<%= ActiveRecord::M
     reversible do |dir|
       dir.up do
         execute <<~SQL
-          UPDATE <%= table_name %> as t
+          UPDATE "<%= full_table_name %>" as t
           SET log_data = logidze_snapshot(<%= logidze_snapshot_parameters %>);
         SQL
       end

data/lib/generators/logidze/model/triggers/logidze.sql

@@ -1,5 +1,5 @@
-CREATE TRIGGER logidze_on_<%= table_name %>
-BEFORE UPDATE OR INSERT ON <%= table_name %> FOR EACH ROW
+CREATE TRIGGER <%= %Q("logidze_on_#{full_table_name}") %>
+BEFORE UPDATE OR INSERT ON <%= %Q("#{full_table_name}") %> FOR EACH ROW
 WHEN (coalesce(current_setting('logidze.disabled', true), '') <> 'on')
 -- Parameters: history_size_limit (integer), timestamp_column (text), filtered_columns (text[]),
 -- include_columns (boolean), debounce_time_ms (integer)

data/lib/logidze/history.rb

@@ -48,7 +48,7 @@ module Logidze
     end
 
     # Return diff from the initial state to specified time or version.
-    # Optional `data` paramater can be used as initial diff state.
+    # Optional `data` parameter can be used as initial diff state.
     def changes_to(time: nil, version: nil, data: {}, from: 0)
       raise ArgumentError, "Time or version must be specified" if time.nil? && version.nil?
 

data/lib/logidze/model.rb

@@ -213,7 +213,7 @@ module Logidze
 
     # Loads log_data field from the database, stores to the attributes hash and returns it
     def reload_log_data
-      self.log_data = self.class.where(self.class.primary_key => id).pluck("#{self.class.table_name}.log_data").first
+      self.log_data = self.class.where(self.class.primary_key => id).pluck("#{self.class.table_name}.log_data".to_sym).first
     end
 
     # Nullify log_data column for a single record
@@ -239,7 +239,7 @@ module Logidze
     end
 
     def apply_column_diff(column, value)
-      return if deleted_column?(column)
+      return if deleted_column?(column) || column == "log_data"
 
       write_attribute column, deserialize_value(column, value)
     end

data/lib/logidze/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Logidze
-  VERSION = "1.1.0"
+  VERSION = "1.2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logidze
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.2
 platform: ruby
 authors:
 - palkan
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-31 00:00:00.000000000 Z
+date: 2022-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -163,6 +163,7 @@ files:
 - lib/generators/logidze/fx_helper.rb
 - lib/generators/logidze/inject_sql.rb
 - lib/generators/logidze/install/USAGE
+- lib/generators/logidze/install/functions/logidze_capture_exception.sql
 - lib/generators/logidze/install/functions/logidze_compact_history.sql
 - lib/generators/logidze/install/functions/logidze_filter_keys.sql
 - lib/generators/logidze/install/functions/logidze_logger.sql
@@ -200,7 +201,7 @@ metadata:
   documentation_uri: http://github.com/palkan/logidze
   homepage_uri: http://github.com/palkan/logidze
   source_code_uri: http://github.com/palkan/logidze
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -215,8 +216,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: PostgreSQL JSONB-based model changes tracking
 test_files: []