better_record 0.1.0 → 0.1.1

data/config/initializers/money_type.rb

@@ -0,0 +1,32 @@
+class MoneyType < ActiveRecord::Type::Value
+  def cast(value)
+    return nil unless value
+    convert_to_money(value)
+  end
+
+  def deserialize(value)
+    super(convert_to_money(value))
+  end
+
+  def serialize(value)
+    new_val = convert_to_money(value)
+    super(new_val ? new_val.value : nil)
+  end
+
+  private
+    def convert_to_money(value)
+      return nil unless value
+      if (!value.kind_of?(Numeric))
+        begin
+          dollars_to_cents = (value.gsub(/\$/, '').presence || 0).to_d * 100
+          StoreAsInt.money(dollars_to_cents.to_i)
+        rescue
+          StoreAsInt::Money.new
+        end
+      else
+        StoreAsInt.money(value)
+      end
+    end
+end
+
+ActiveRecord::Type.register(:money_column, MoneyType)

data/config/routes.rb

@@ -1,2 +1,4 @@
 BetterRecord::Engine.routes.draw do
+  root to: 'table_sizes#index'
+  resources :table_sizes, only: [ :index, :show ]
 end

data/db/migrate/20180725160802_create_better_record_db_functions.rb

@@ -0,0 +1,94 @@
+class CreateBetterRecordDBFunctions < ActiveRecord::Migration[5.2]
+  def up
+    execute "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
+    execute "CREATE EXTENSION IF NOT EXISTS btree_gin;"
+    execute "CREATE EXTENSION IF NOT EXISTS pgcrypto;"
+
+    sql = ""
+    source = File.new(BetterRecord::Engine.root.join('db', 'postgres-audit-trigger.psql'), "r")
+    while (line = source.gets)
+      sql << line.gsub(/SELECTED_SCHEMA_NAME/, BetterRecord.db_audit_schema)
+    end
+    source.close
+
+    execute sql
+
+    execute <<-SQL
+      CREATE or REPLACE FUNCTION public.temp_table_exists( varchar)
+      RETURNS pg_catalog.bool AS
+      $$
+        BEGIN
+          /* check the table exist in database and is visible*/
+          PERFORM n.nspname, c.relname
+          FROM pg_catalog.pg_class c
+          LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+          WHERE n.nspname LIKE 'pg_temp_%' AND pg_catalog.pg_table_is_visible(c.oid)
+          AND relname = $1;
+
+          IF FOUND THEN
+            RETURN TRUE;
+          ELSE
+            RETURN FALSE;
+          END IF;
+
+        END;
+      $$
+      LANGUAGE 'plpgsql' VOLATILE
+    SQL
+
+    execute <<-SQL
+      CREATE OR REPLACE FUNCTION hash_password(password text)
+        RETURNS text AS
+      $BODY$
+      BEGIN
+        password = crypt(password, gen_salt('bf', 8));
+
+        RETURN password;
+      END;
+      $BODY$
+
+      LANGUAGE plpgsql;
+    SQL
+
+    execute <<-SQL
+      CREATE OR REPLACE FUNCTION validate_email(email text)
+        RETURNS text AS
+      $BODY$
+      BEGIN
+        IF email IS NOT NULL THEN
+          IF email !~* '\\A[^@\\s\\;]+@[^@\\s\\;]+\\.[^@\\s\\;]+\\Z' THEN
+            RAISE EXCEPTION 'Invalid E-mail format %', email
+                USING HINT = 'Please check your E-mail format.';
+          END IF ;
+          email = lower(email);
+        END IF ;
+
+        RETURN email;
+      END;
+      $BODY$
+      LANGUAGE plpgsql;
+    SQL
+
+    execute <<-SQL
+      CREATE OR REPLACE FUNCTION valid_email_trigger()
+        RETURNS TRIGGER AS
+      $BODY$
+      BEGIN
+        NEW.email = validate_email(NEW.email);
+
+        RETURN NEW;
+      END;
+      $BODY$
+      LANGUAGE plpgsql;
+    SQL
+
+  end
+
+  def down
+    execute 'DROP FUNCTION IF EXISTS valid_email_trigger();'
+    execute 'DROP FUNCTION IF EXISTS validate_email();'
+    execute 'DROP FUNCTION IF EXISTS temp_table_exists();'
+    execute "DROP FUNCTION IF EXISTS hash_password();"
+    execute "DROP SCHEMA IF EXISTS #{BetterRecord.db_audit_schema} CASCADE;"
+  end
+end

data/db/migrate/20180725201614_create_better_record_table_sizes.rb

@@ -0,0 +1,24 @@
+class CreateBetterRecordTableSizes < ActiveRecord::Migration[5.2]
+  def change
+    create_table "#{BetterRecord.db_audit_schema}.table_sizes", {id: false} do |t|
+      t.integer :oid, limit: 8
+      t.string :schema
+      t.string :name
+      t.float :apx_row_count
+      t.integer :total_bytes, limit: 8
+      t.integer :idx_bytes, limit: 8
+      t.integer :toast_bytes, limit: 8
+      t.integer :tbl_bytes, limit: 8
+      t.text :total
+      t.text :idx
+      t.text :toast
+      t.text :tbl
+    end
+
+    reversible do |d|
+      d.up do
+        execute "ALTER TABLE #{BetterRecord.db_audit_schema}.table_sizes ADD PRIMARY KEY (oid);"
+      end
+    end
+  end
+end

data/db/postgres-audit-trigger.psql

@@ -0,0 +1,347 @@
+-- An audit history is important on most tables. Provide an audit trigger that logs to
+-- a dedicated audit table for the major relations.
+--
+-- This file should be generic and not depend on application roles or structures,
+-- as it's being listed here:
+--
+--  https://wiki.postgresql.org/wiki/Audit_trigger_91plus
+--
+-- This trigger was originally based on
+--   http://wiki.postgresql.org/wiki/Audit_trigger
+-- but has been completely rewritten.
+--
+-- Should really be converted into a relocatable EXTENSION, with control and upgrade files.
+
+CREATE EXTENSION IF NOT EXISTS hstore;
+
+CREATE SCHEMA SELECTED_SCHEMA_NAME;
+REVOKE ALL ON SCHEMA SELECTED_SCHEMA_NAME FROM public;
+
+COMMENT ON SCHEMA SELECTED_SCHEMA_NAME IS 'Out-of-table audit/history logging tables and trigger functions';
+
+--
+-- Audited data. Lots of information is available, it's just a matter of how much
+-- you really want to record. See:
+--
+--   http://www.postgresql.org/docs/9.1/static/functions-info.html
+--
+-- Remember, every column you add takes up more audit table space and slows audit
+-- inserts.
+--
+-- Every index you add has a big impact too, so avoid adding indexes to the
+-- audit table unless you REALLY need them. The hstore GIST indexes are
+-- particularly expensive.
+--
+-- It is sometimes worth copying the audit table, or a coarse subset of it that
+-- you're interested in, into a temporary table where you CREATE any useful
+-- indexes and do your analysis.
+--
+CREATE TABLE SELECTED_SCHEMA_NAME.logged_actions (
+  event_id bigserial primary key,
+  schema_name text not null,
+  table_name text not null,
+  relid oid not null,
+  session_user_name text,
+  app_user_id integer,
+  app_user_type text,
+  app_ip_address inet,
+  action_tstamp_tx TIMESTAMP WITH TIME ZONE NOT NULL,
+  action_tstamp_stm TIMESTAMP WITH TIME ZONE NOT NULL,
+  action_tstamp_clk TIMESTAMP WITH TIME ZONE NOT NULL,
+  transaction_id bigint,
+  application_name text,
+  client_addr inet,
+  client_port integer,
+  client_query text,
+  action TEXT NOT NULL CHECK (action IN ('I','D','U', 'T')),
+  row_id bigint,
+  row_data hstore,
+  changed_fields hstore,
+  statement_only boolean not null
+);
+
+REVOKE ALL ON SELECTED_SCHEMA_NAME.logged_actions FROM public;
+
+COMMENT ON TABLE SELECTED_SCHEMA_NAME.logged_actions IS 'History of auditable actions on audited tables, from SELECTED_SCHEMA_NAME.if_modified_func()';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.event_id IS 'Unique identifier for each auditable event';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.schema_name IS 'Database schema audited table for this event is in';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.table_name IS 'Non-schema-qualified table name of table event occured in';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.relid IS 'Table OID. Changes with drop/create. Get with ''tablename''::regclass';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.session_user_name IS 'Login / session user whose statement caused the audited event';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.app_user_id IS 'Application-provided polymorphic user id';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.app_user_type IS 'Application-provided polymorphic user type';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.app_ip_address IS 'Application-provided ip address of user whose statement caused the audited event';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.action_tstamp_tx IS 'Transaction start timestamp for tx in which audited event occurred';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.action_tstamp_stm IS 'Statement start timestamp for tx in which audited event occurred';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.action_tstamp_clk IS 'Wall clock time at which audited event''s trigger call occurred';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.transaction_id IS 'Identifier of transaction that made the change. May wrap, but unique paired with action_tstamp_tx.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.client_addr IS 'IP address of client that issued query. Null for unix domain socket.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.client_port IS 'Remote peer IP port address of client that issued query. Undefined for unix socket.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.client_query IS 'Top-level query that caused this auditable event. May be more than one statement.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.application_name IS 'Application name set when this audit event occurred. Can be changed in-session by client.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.action IS 'Action type; I = insert, D = delete, U = update, T = truncate';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.row_id IS 'Record primary_key. Null for statement-level trigger. Prefers NEW.id if exists';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.row_data IS 'Record value. Null for statement-level trigger. For INSERT this is the new tuple. For DELETE and UPDATE it is the old tuple.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.changed_fields IS 'New values of fields changed by UPDATE. Null except for row-level UPDATE events.';
+COMMENT ON COLUMN SELECTED_SCHEMA_NAME.logged_actions.statement_only IS '''t'' if audit event is from an FOR EACH STATEMENT trigger, ''f'' for FOR EACH ROW';
+
+CREATE INDEX logged_actions_relid_idx ON SELECTED_SCHEMA_NAME.logged_actions(relid);
+CREATE INDEX logged_actions_action_tstamp_tx_stm_idx ON SELECTED_SCHEMA_NAME.logged_actions(action_tstamp_stm);
+CREATE INDEX logged_actions_action_idx ON SELECTED_SCHEMA_NAME.logged_actions(action);
+CREATE INDEX logged_actions_row_id_idx ON SELECTED_SCHEMA_NAME.logged_actions(row_id);
+
+CREATE OR REPLACE FUNCTION SELECTED_SCHEMA_NAME.if_modified_func()
+RETURNS TRIGGER AS
+$$
+  DECLARE
+    audit_row SELECTED_SCHEMA_NAME.logged_actions;
+    include_values boolean;
+    log_diffs boolean;
+    h_old hstore;
+    h_new hstore;
+    user_row record;
+    excluded_cols text[] = ARRAY[]::text[];
+    pk_val_query text;
+  BEGIN
+    IF TG_WHEN <> 'AFTER' THEN
+      RAISE EXCEPTION 'SELECTED_SCHEMA_NAME.if_modified_func() may only run as an AFTER trigger';
+    END IF;
+
+    audit_row = ROW(
+      nextval('SELECTED_SCHEMA_NAME.logged_actions_event_id_seq'), -- event_id
+      TG_TABLE_SCHEMA::text,                        -- schema_name
+      TG_TABLE_NAME::text,                          -- table_name
+      TG_RELID,                                     -- relation OID for much quicker searches
+      session_user::text,                           -- session_user_name
+      NULL, NULL, NULL,                             -- app_user_id, app_user_type, app_ip_address
+      current_timestamp,                            -- action_tstamp_tx
+      statement_timestamp(),                        -- action_tstamp_stm
+      clock_timestamp(),                            -- action_tstamp_clk
+      txid_current(),                               -- transaction ID
+      current_setting('application_name'),          -- client application
+      inet_client_addr(),                           -- client_addr
+      inet_client_port(),                           -- client_port
+      current_query(),                              -- top-level query or queries (if multistatement) from client
+      substring(TG_OP,1,1),                         -- action
+      NULL, NULL, NULL,                             -- row_id, row_data, changed_fields
+      'f'                                           -- statement_only
+    );
+
+    IF NOT TG_ARGV[0]::boolean IS DISTINCT FROM 'f'::boolean THEN
+      audit_row.client_query = NULL;
+    END IF;
+
+    IF ((TG_ARGV[1] IS NOT NULL) AND (TG_LEVEL = 'ROW')) THEN
+      pk_val_query = 'SELECT $1.' || quote_ident(TG_ARGV[1]::text);
+
+      IF (TG_OP IS DISTINCT FROM 'DELETE') THEN
+        EXECUTE pk_val_query INTO audit_row.row_id USING NEW;
+      END IF;
+
+      IF audit_row.row_id IS NULL THEN
+        EXECUTE pk_val_query INTO audit_row.row_id USING OLD;
+      END IF;
+    END IF;
+
+    IF TG_ARGV[2] IS NOT NULL THEN
+      excluded_cols = TG_ARGV[2]::text[];
+    END IF;
+
+
+
+    IF (TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW') THEN
+      audit_row.row_data = hstore(OLD.*) - excluded_cols;
+      audit_row.changed_fields =  (hstore(NEW.*) - audit_row.row_data) - excluded_cols;
+      IF audit_row.changed_fields = hstore('') THEN
+        -- All changed fields are ignored. Skip this update.
+        RETURN NULL;
+      END IF;
+    ELSIF (TG_OP = 'DELETE' AND TG_LEVEL = 'ROW') THEN
+      audit_row.row_data = hstore(OLD.*) - excluded_cols;
+    ELSIF (TG_OP = 'INSERT' AND TG_LEVEL = 'ROW') THEN
+      audit_row.row_data = hstore(NEW.*) - excluded_cols;
+    ELSIF (TG_LEVEL = 'STATEMENT' AND TG_OP IN ('INSERT','UPDATE','DELETE','TRUNCATE')) THEN
+      audit_row.statement_only = 't';
+    ELSE
+      RAISE EXCEPTION '[SELECTED_SCHEMA_NAME.if_modified_func] - Trigger func added as trigger for unhandled case: %, %',TG_OP, TG_LEVEL;
+      RETURN NULL;
+    END IF;
+
+    -- inject app_user data into audit
+    BEGIN
+      PERFORM
+      n.nspname, c.relname
+      FROM
+      pg_catalog.pg_class c
+      LEFT JOIN
+      pg_catalog.pg_namespace n
+      ON n.oid = c.relnamespace
+      WHERE
+      n.nspname like 'pg_temp_%'
+      AND
+      c.relname = '_app_user';
+
+      IF FOUND THEN
+      FOR user_row IN SELECT * FROM _app_user LIMIT 1 LOOP
+        audit_row.app_user_id = user_row.user_id;
+        audit_row.app_user_type = user_row.user_type;
+        audit_row.app_ip_address = user_row.ip_address;
+      END LOOP;
+      END IF;
+    END;
+    -- end app_user data
+
+    INSERT INTO SELECTED_SCHEMA_NAME.logged_actions VALUES (audit_row.*);
+    RETURN NULL;
+  END;
+$$
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog, public;
+
+
+COMMENT ON FUNCTION SELECTED_SCHEMA_NAME.if_modified_func() IS
+$$
+  Track changes to a table at the statement and/or row level.
+
+  Optional parameters to trigger in CREATE TRIGGER call:
+
+  param 0: boolean, whether to log the query text. Default 't'.
+
+  param 1: text, primary_key_column of audited table if bigint.
+
+  param 2: text[], columns to ignore in updates. Default [].
+
+       Updates to ignored cols are omitted from changed_fields.
+
+       Updates with only ignored cols changed are not inserted
+       into the audit log.
+
+       Almost all the processing work is still done for updates
+       that ignored. If you need to save the load, you need to use
+       WHEN clause on the trigger instead.
+
+       No warning or error is issued if ignored_cols contains columns
+       that do not exist in the target table. This lets you specify
+       a standard set of ignored columns.
+
+  There is no parameter to disable logging of values. Add this trigger as
+  a 'FOR EACH STATEMENT' rather than 'FOR EACH ROW' trigger if you do not
+  want to log row values.
+
+  Note that the user name logged is the login role for the session. The audit trigger
+  cannot obtain the active role because it is reset by the SECURITY DEFINER invocation
+  of the audit trigger its self.
+$$;
+
+
+CREATE OR REPLACE FUNCTION SELECTED_SCHEMA_NAME.get_primary_key_column(target_table text)
+RETURNS text AS
+$$
+  DECLARE
+    _pk_query_text text;
+    _pk_column_name text;
+  BEGIN
+    _pk_query_text =  'SELECT a.attname ' ||
+                      'FROM   pg_index i ' ||
+                      'JOIN   pg_attribute a ON a.attrelid = i.indrelid ' ||
+                      '                    AND a.attnum = ANY(i.indkey) ' ||
+                      'WHERE  i.indrelid = ' || quote_literal(target_table::TEXT) || '::regclass ' ||
+                      'AND    i.indisprimary ' ||
+                      'AND format_type(a.atttypid, a.atttypmod) = ' || quote_literal('bigint'::TEXT) ||
+                      'LIMIT 1';
+
+    EXECUTE _pk_query_text INTO _pk_column_name;
+    raise notice 'Value %', _pk_column_name;
+    return _pk_column_name;
+  END;
+$$
+LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION SELECTED_SCHEMA_NAME.get_primary_key_column(text) IS
+$$
+  Get primary key column name if single PK and type bigint.
+
+  Arguments:
+      target_table:   Table name, schema qualified if not on search_path
+$$;
+
+CREATE OR REPLACE FUNCTION SELECTED_SCHEMA_NAME.audit_table(target_table regclass, audit_rows boolean, audit_query_text boolean, ignored_cols text[])
+RETURNS void AS
+$$
+  DECLARE
+    stm_targets text = 'INSERT OR UPDATE OR DELETE OR TRUNCATE';
+    _q_txt text;
+    _pk_column_name text;
+    _pk_column_snip text;
+    _ignored_cols_snip text = '';
+  BEGIN
+    EXECUTE 'DROP TRIGGER IF EXISTS audit_trigger_row ON ' || quote_ident(target_table::TEXT);
+    EXECUTE 'DROP TRIGGER IF EXISTS audit_trigger_stm ON ' || quote_ident(target_table::TEXT);
+
+    IF audit_rows THEN
+      _pk_column_name = SELECTED_SCHEMA_NAME.get_primary_key_column(target_table::TEXT);
+
+      IF _pk_column_name IS NOT NULL THEN
+        _pk_column_snip = ', ' || quote_literal(_pk_column_name);
+      ELSE
+        _pk_column_snip = ', NULL';
+      END IF;
+
+      IF array_length(ignored_cols,1) > 0 THEN
+        _ignored_cols_snip = ', ' || quote_literal(ignored_cols);
+      END IF;
+      _q_txt = 'CREATE TRIGGER audit_trigger_row AFTER INSERT OR UPDATE OR DELETE ON ' ||
+          quote_ident(target_table::TEXT) ||
+          ' FOR EACH ROW EXECUTE PROCEDURE SELECTED_SCHEMA_NAME.if_modified_func(' ||
+          quote_literal(audit_query_text) || _pk_column_snip || _ignored_cols_snip || ');';
+      RAISE NOTICE '%',_q_txt;
+      EXECUTE _q_txt;
+      stm_targets = 'TRUNCATE';
+    ELSE
+    END IF;
+
+    _q_txt = 'CREATE TRIGGER audit_trigger_stm AFTER ' || stm_targets || ' ON ' ||
+        target_table ||
+        ' FOR EACH STATEMENT EXECUTE PROCEDURE SELECTED_SCHEMA_NAME.if_modified_func('||
+        quote_literal(audit_query_text) || ');';
+    RAISE NOTICE '%',_q_txt;
+    EXECUTE _q_txt;
+
+  END;
+$$
+LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION SELECTED_SCHEMA_NAME.audit_table(regclass, boolean, boolean, text[]) IS
+$$
+  Add auditing support to a table.
+
+  Arguments:
+      target_table:   Table name, schema qualified if not on search_path
+      audit_rows:     Record each row change, or only audit at a statement level
+      audit_query_text: Record the text of the client query that triggered the audit event?
+      ignored_cols:   Columns to exclude from update diffs, ignore updates that change only ignored cols.
+$$;
+
+-- Pg doesn't allow variadic calls with 0 params, so provide a wrapper
+CREATE OR REPLACE FUNCTION SELECTED_SCHEMA_NAME.audit_table(target_table regclass, audit_rows boolean, audit_query_text boolean)
+RETURNS void AS
+$$
+  SELECT SELECTED_SCHEMA_NAME.audit_table($1, $2, $3, ARRAY[]::text[]);
+$$
+LANGUAGE SQL;
+
+-- And provide a convenience call wrapper for the simplest case
+-- of row-level logging with no excluded cols and query logging enabled.
+--
+CREATE OR REPLACE FUNCTION SELECTED_SCHEMA_NAME.audit_table(target_table regclass)
+RETURNS void AS
+$$
+  SELECT SELECTED_SCHEMA_NAME.audit_table($1, BOOLEAN 't', BOOLEAN 't');
+$$
+LANGUAGE SQL;
+
+COMMENT ON FUNCTION SELECTED_SCHEMA_NAME.audit_table(regclass) IS
+$$
+  Add auditing support to the given table. Row-level changes will be logged with full client query text. No cols are ignored.
+$$;