postqueue 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ab5c97e45f6171bf141d58d1cd8ce8c5524f5b2
-  data.tar.gz: 0a1b8ef785008bec26a6907024617a4ef4a82a3e
+  metadata.gz: 169eb71de24fab5bde343459df49bf1395881fc4
+  data.tar.gz: fb06468f9fe5412dc655188823d5544be0ba9d48
 SHA512:
-  metadata.gz: 31fad4af31402d2084e07c0b60baa14bce82a53efba6714056956f52da1360cfd5b92a6a0f6599789f687290ec743f867604346c4bc7a16c930aa52a69071585
-  data.tar.gz: a7c2117c1abdee7840cfa66714e99ad15bd8a02456454d4f4d44ce9a00e019df5224e722c7e4f415116331bb3c3e2a08bb14b77d0204dd9c51a0941dcaaa305a
+  metadata.gz: 5589efb14a39dedb9b56c207eaffdbef1e8533cc9717991a0b8d2cdb12a0966b15cbf712b7720ff2df3c1b0e75b7f66f9402b8bfa9eeb3b62308cff3cd6ed6a6
+  data.tar.gz: 26cf306354c7a91c9c1bb172cca2aaf2639195aa25c0b9005d4244335f9a8e46c5a85848fa78afe869addb0e0d03423eba53c1bcebb0f3128b3d00abae3bd192

data/lib/postqueue/item/inserter.rb

@@ -1,4 +1,5 @@
 require "active_record"
+require "simple/sql"
 
 module Postqueue
   #
@@ -7,52 +8,11 @@ module Postqueue
   # This source file provides multiple implementations to insert Postqueue::Items.
   # Which one will be used depends on the "extend XXXInserter" line below.
   class Item < ActiveRecord::Base
-    module ActiveRecordInserter
-      def insert_item(op:, entity_id:)
-        create!(op: op, entity_id: entity_id)
-      end
+    def self.insert_item(op:, entity_id:)
+      # In contrast to ActiveRecord, which clocks in around 600µs per item,
+      # Simple::SQL's insert only takes 100µs per item. Using prepared
+      # statements would further reduce the runtime to 50µs
+      ::Simple::SQL.insert table_name, op: op, entity_id: entity_id
     end
-
-    module RawInserter
-      def insert_sql
-        "INSERT INTO #{table_name}(op, entity_id) VALUES($1, $2)"
-      end
-
-      def insert_item(op:, entity_id:)
-        connection.raw_connection.exec_params(insert_sql, [op, entity_id])
-      end
-    end
-
-    module PreparedRawInserter
-      def insert_sql
-        "INSERT INTO #{table_name}(op, entity_id) VALUES($1, $2)"
-      end
-
-      def prepared_inserter_statement(raw_connection)
-        @prepared_inserter_statements ||= {}
-
-        # a prepared connection is PER DATABASE CONNECTION. It is not shared across
-        # connections, and it is not per thread, since a Thread might use different
-        # connections during its lifetime.
-        @prepared_inserter_statements[raw_connection.object_id] ||= create_prepared_inserter_statement(raw_connection)
-      end
-
-      # prepares the INSERT statement, and returns its name
-      def create_prepared_inserter_statement(raw_connection)
-        name = "postqueue-insert-#{table_name}-#{raw_connection.object_id}"
-        raw_connection.prepare(name, insert_sql)
-        name
-      end
-
-      def insert_item(op:, entity_id:)
-        raw_connection = connection.raw_connection
-        statement_name = prepared_inserter_statement(raw_connection)
-        raw_connection.exec_prepared(statement_name, [op, entity_id])
-      end
-    end
-
-    # extend ActiveRecordInserter   # 600µs per item
-    extend RawInserter              # 100µs per item
-    # extend PreparedRawInserter    # 50µs per item
   end
 end

data/lib/postqueue/tracker/tracker.sql

@@ -0,0 +1,198 @@
+-- An tracker history is important on most tables. Provide an tracker trigger that logs to
+-- a dedicated tracker table for the major relations.
+--
+-- This file should be generic and not depend on application roles or structures,
+-- as it's being listed here:
+--
+--    https://wiki.postgresql.org/wiki/Audit_trigger_91plus    
+--
+-- This trigger was originally based on
+--   http://wiki.postgresql.org/wiki/Audit_trigger
+-- but has been completely rewritten.
+--
+-- Should really be converted into a relocatable EXTENSION, with control and upgrade files.
+
+CREATE EXTENSION IF NOT EXISTS hstore;
+
+<% if true %>
+DROP SCHEMA IF EXISTS tracker CASCADE;
+<% end %>
+CREATE SCHEMA tracker;
+REVOKE ALL ON SCHEMA tracker FROM public;
+
+COMMENT ON SCHEMA tracker IS 'Out-of-table tracing';
+
+CREATE TYPE tracker.actions AS ENUM(
+  'INSERT', 'UPDATE', 'DELETE'
+);
+
+--
+-- Audited data. Lots of information is available, it's just a matter of how much
+-- you really want to record. See:
+--
+--   http://www.postgresql.org/docs/9.1/static/functions-info.html
+--
+-- Remember, every column you add takes up more tracker table space and slows tracker
+-- inserts.
+--
+-- Every index you add has a big impact too, so avoid adding indexes to the
+-- tracker table unless you REALLY need them. The hstore GIST indexes are
+-- particularly expensive.
+--
+-- It is sometimes worth copying the tracker table, or a coarse subset of it that
+-- you're interested in, into a temporary table where you CREATE any useful
+-- indexes and do your analysis.
+--
+CREATE TABLE tracker.events (
+  id bigserial primary key,                     -- id IS 'Unique identifier for each tracked event';
+  row_data hstore,                              -- row_data IS 'Record value. For INSERT this is the new tuple. For DELETE and UPDATE it is the old tuple.';
+  changed_fields hstore,                        -- changed_fields IS 'New values of fields changed by UPDATE. Null except for UPDATE events.';
+  created_at TIMESTAMP WITH TIME ZONE NOT NULL, -- created_at IS 'Wall clock time at which tracked event''s trigger call occurred';
+  checked_out_at TIMESTAMP WITH TIME ZONE       -- checked_out_at IS 'Wall clock time at which event is checked out for processing';
+);
+
+
+REVOKE ALL ON tracker.events FROM public;
+
+/*
+
+[TODO] - add comments on columns
+
+*/
+
+COMMENT ON TABLE tracker.events IS 'History of tracked actions on tracked tables, from tracker.if_modified_func()';
+-- COMMENT ON COLUMN tracker.events.session_user_name IS 'Login / session user whose statement caused the tracked event';
+
+/*
+
+[TODO] - create indices for common search patterns
+
+*/
+
+-- CREATE INDEX events_table_oid_idx ON tracker.events(table_oid);
+-- CREATE INDEX events_action_idx ON tracker.events(action);
+
+/* [TODO] - change checked_out_at to default to 'infinity' */
+
+
+/*
+
+Track changes to a table at row level.
+
+Optional parameters to trigger in CREATE TRIGGER call:
+
+param 0: text, name of primary key column;
+param 1: text[], columns to ignore in updates. Default [].
+
+Updates to ignored cols are omitted from changed_fields.
+
+Updates with only ignored cols changed are not inserted
+into the tracker log.
+
+Note that the user name logged is the login role for the session.
+The tracker trigger cannot obtain the active role because it is reset
+by the SECURITY DEFINER invocation of the tracker trigger its self.
+
+*/
+
+CREATE OR REPLACE FUNCTION tracker.if_modified_func() RETURNS TRIGGER AS $body$
+DECLARE
+  event fq_table_name;
+  h_old hstore;
+  h_new hstore;
+  entity_pkey_name text;
+BEGIN
+  IF TG_WHEN <> 'AFTER' THEN
+    RAISE EXCEPTION 'tracker.if_modified_func() may only run as an AFTER trigger';
+  END IF;
+
+  -- get args
+
+  entity_pkey_name = TG_ARGV[0];
+
+  -- fill row
+
+  IF (TG_OP = 'UPDATE') THEN
+    event.new_fields = jsonb(NEW.*);
+    event.old_fields = jsonb(OLD.*);
+    event.entity_id  = event.new_fields->entity_pkey_name
+  ELSIF (TG_OP = 'DELETE') THEN
+    event.old_fields = jsonb(OLD.*);
+    event.entity_id  = event.old_fields->entity_pkey_name
+  ELSIF (TG_OP = 'INSERT') THEN
+    event.new_fields = jsonb(NEW.*);
+    event.entity_id  = event.new_fields->entity_pkey_name
+  END IF;
+
+  INSERT INTO tracker.events VALUES (event.*);
+  RETURN NULL;
+END;
+$body$
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog, public;
+
+/*
+
+Add tracking support to a table.
+
+Arguments:
+   target_table:     Table name, schema qualified if not on search_path
+   primary_key_name: Name of primary key column
+   ignored_cols:     Columns to exclude from update diffs, ignore updates that change only ignored cols.
+
+*/
+
+CREATE OR REPLACE FUNCTION tracker.track_table(target_table regclass, entity_pkey_name text, ignored_cols text[]) 
+  RETURNS void AS $body$
+DECLARE
+  _q_txt text;
+  _ignored_cols_snip text = '';
+BEGIN
+  IF array_length(ignored_cols,1) > 0 THEN
+    _ignored_cols_snip = ', ' || quote_literal(ignored_cols);
+  END IF;
+
+  EXECUTE 'DROP TRIGGER IF EXISTS track_trigger_row ON ' || quote_ident(target_table::TEXT);
+
+  _q_txt = 'CREATE TRIGGER track_trigger_row AFTER INSERT OR UPDATE OR DELETE ON ' || 
+              quote_ident(target_table::TEXT) || 
+              ' FOR EACH ROW EXECUTE PROCEDURE tracker.if_modified_func(' ||
+              quote_literal(entity_pkey_name) || 
+              _ignored_cols_snip || ');';
+  EXECUTE _q_txt;
+END;
+$body$
+language 'plpgsql';
+
+
+/*
+
+Add tracking support to the given table. No cols are ignored. (Shortcut)
+
+*/
+
+CREATE OR REPLACE FUNCTION tracker.track_table(target_table regclass) RETURNS void AS $body$
+SELECT tracker.track_table($1, 'id', ARRAY[]::text[]);
+$body$ LANGUAGE 'sql';
+
+COMMENT ON FUNCTION tracker.track_table(regclass) IS $body$
+$body$;
+
+
+--
+------------------------------------------------------------------------------------
+--
+
+-- SELECT tracker.track_table('public.posts');
+
+--SELECT tracker.track_table('public.users', 'id', array['created_at','updated_at']);
+-- -- SELECT tracker.track_table('public.users', ARRAY[]::text[]);
+-- -- SELECT tracker.track_table('public.users');
+--
+-- INSERT INTO users (email, created_at, updated_at) VALUES('me@mo' || (1000 * random())::integer , NOW(), NOW());
+-- UPDATE users SET username='mimi' WHERE email LIKE '%mo%';
+-- UPDATE users SET username='momo' WHERE email LIKE '%mo%';
+-- DELETE FROM users WHERE email LIKE '%mo%';
+--
+-- SELECT table_schema || '.' || table_name || '.' || entity_pkey, action, row_data, changed_fields, created_at FROM tracker.events;

data/lib/postqueue/version.rb

@@ -1,3 +1,3 @@
 module Postqueue
-  VERSION = "0.6.1"
+  VERSION = "0.7.0"
 end

data/spec/spec_helper.rb

@@ -11,7 +11,10 @@ end
 require "postqueue"
 require "./spec/support/configure_active_record"
 
-Postqueue.logger = Logger.new(File.open("log/test.log", "a"))
+logger = Logger.new(File.open("log/test.log", "a"))
+logger.level = Logger::INFO
+
+Simple::SQL.logger = Postqueue.logger = logger
 
 RSpec.configure do |config|
   config.run_all_when_everything_filtered = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postqueue
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-15 00:00:00.000000000 Z
+date: 2018-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simple-sql
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.6
 - !ruby/object:Gem::Dependency
   name: table_print
   requirement: !ruby/object:Gem::Requirement
@@ -177,6 +191,7 @@ files:
 - lib/postqueue/queue/runner.rb
 - lib/postqueue/queue/select_and_lock.rb
 - lib/postqueue/queue/timing.rb
+- lib/postqueue/tracker/tracker.sql
 - lib/postqueue/version.rb
 - lib/tracker.rb
 - lib/tracker/advisory_lock.rb