postqueue 0.2.1 → 0.4.0

data/lib/tracker/migration.rb

@@ -0,0 +1,23 @@
+module Tracker
+  module Migration
+    class Options < OpenStruct
+      def render(erb)
+        erb.result(binding)
+      end
+    end
+
+    def migrate!
+      template = File.read(File.dirname(__FILE__) + "/tracker.sql")
+      renderer = ERB.new(template)
+
+      options = Options.new(reinstall: false)
+      sql = options.render(renderer)
+      ActiveRecord::Base.connection.execute sql
+    end
+
+    def track_table!(table)
+      sql = "SELECT tracker.track_table('#{table}', 'id', array['created_at','updated_at', 'tsv', 'pg_search'])"
+      ActiveRecord::Base.connection.execute sql
+    end
+  end
+end

data/lib/tracker/registry.rb

@@ -0,0 +1,45 @@
+module Tracker
+  module Registry
+    extend self
+
+    def reset!
+      @registrations = nil
+    end
+
+    private
+
+    def registrations
+      @registrations ||= Hash.new { |h, k| h[k] = [] }
+    end
+
+    def register_callback(event, &proc)
+      STDERR.puts "Starting to track #{event.inspect}"
+      callbacks(event) << proc
+    end
+
+    def callbacks(event)
+      registrations[event]
+    end
+
+    public
+
+    def on(event, &proc)
+      expect! event => /(insert|delete|update)$/
+      register_callback event, &proc
+    end
+
+    def track(table, &proc)
+      register_callback table, &proc
+    end
+
+    def tracks?(table)
+      !callbacks(table).empty?
+    end
+
+    def publish!(event_name, *args)
+      callbacks(event_name).each do |callback|
+        callback.call(*args)
+      end
+    end
+  end
+end

data/lib/tracker/tracker.sql

@@ -0,0 +1,231 @@
+-- An tracker history is important on most tables. Provide an tracker trigger that logs to
+-- a dedicated tracker table for the major relations.
+--
+-- This file should be generic and not depend on application roles or structures,
+-- as it's being listed here:
+--
+--    https://wiki.postgresql.org/wiki/Audit_trigger_91plus    
+--
+-- This trigger was originally based on
+--   http://wiki.postgresql.org/wiki/Audit_trigger
+-- but has been completely rewritten.
+--
+-- Should really be converted into a relocatable EXTENSION, with control and upgrade files.
+
+CREATE EXTENSION IF NOT EXISTS hstore;
+
+<% if true %>
+DROP SCHEMA IF EXISTS tracker CASCADE;
+<% end %>
+CREATE SCHEMA tracker;
+REVOKE ALL ON SCHEMA tracker FROM public;
+
+COMMENT ON SCHEMA tracker IS 'Out-of-table tracing';
+
+CREATE TYPE tracker.actions AS ENUM(
+  'INSERT', 'UPDATE', 'DELETE'
+);
+
+--
+-- Audited data. Lots of information is available, it's just a matter of how much
+-- you really want to record. See:
+--
+--   http://www.postgresql.org/docs/9.1/static/functions-info.html
+--
+-- Remember, every column you add takes up more tracker table space and slows tracker
+-- inserts.
+--
+-- Every index you add has a big impact too, so avoid adding indexes to the
+-- tracker table unless you REALLY need them. The hstore GIST indexes are
+-- particularly expensive.
+--
+-- It is sometimes worth copying the tracker table, or a coarse subset of it that
+-- you're interested in, into a temporary table where you CREATE any useful
+-- indexes and do your analysis.
+--
+CREATE TABLE tracker.events (
+  id bigserial primary key,                     -- id IS 'Unique identifier for each tracked event';
+  table_oid oid not null,                       -- table_oid IS 'Table OID. Changes with drop/create. Get with ''tablename''::regclass';
+  table_schema text not null,                   -- table_schema IS 'Database schema tracked table for this event is in';
+  table_name text not null,                     -- table_name IS 'Non-schema-qualified table name of table event occured in';
+  action tracker.actions,                       -- event's action: INSERT, UPDATE, or DELETE 
+  entity_pkey text not null,                    -- entity_pkey IS 'ROW primary key.';
+  row_data hstore,                              -- row_data IS 'Record value. For INSERT this is the new tuple. For DELETE and UPDATE it is the old tuple.';
+  changed_fields hstore,                        -- changed_fields IS 'New values of fields changed by UPDATE. Null except for UPDATE events.';
+  created_at TIMESTAMP WITH TIME ZONE NOT NULL, -- created_at IS 'Wall clock time at which tracked event''s trigger call occurred';
+  checked_out_at TIMESTAMP WITH TIME ZONE       -- checked_out_at IS 'Wall clock time at which event is checked out for processing';
+
+  -- session_user_name text,                       -- session_user_name IS 'Login / session user whose statement caused the tracked event';
+);
+
+
+REVOKE ALL ON tracker.events FROM public;
+
+/*
+
+[TODO] - add comments on columns
+
+*/
+
+COMMENT ON TABLE tracker.events IS 'History of tracked actions on tracked tables, from tracker.if_modified_func()';
+-- COMMENT ON COLUMN tracker.events.session_user_name IS 'Login / session user whose statement caused the tracked event';
+
+/*
+
+[TODO] - create indices for common search patterns
+
+*/
+
+-- CREATE INDEX events_table_oid_idx ON tracker.events(table_oid);
+-- CREATE INDEX events_action_idx ON tracker.events(action);
+
+/* [TODO] - change checked_out_at to default to 'infinity' */
+
+
+/*
+
+Track changes to a table at row level.
+
+Optional parameters to trigger in CREATE TRIGGER call:
+
+param 0: text, name of primary key column;
+param 1: text[], columns to ignore in updates. Default [].
+
+Updates to ignored cols are omitted from changed_fields.
+
+Updates with only ignored cols changed are not inserted
+into the tracker log.
+
+Note that the user name logged is the login role for the session.
+The tracker trigger cannot obtain the active role because it is reset
+by the SECURITY DEFINER invocation of the tracker trigger its self.
+
+*/
+
+CREATE OR REPLACE FUNCTION tracker.if_modified_func() RETURNS TRIGGER AS $body$
+DECLARE
+  event tracker.events;
+  h_old hstore;
+  h_new hstore;
+  excluded_cols text[] = ARRAY[]::text[];
+  entity_pkey_name text;
+BEGIN
+  IF TG_WHEN <> 'AFTER' THEN
+    RAISE EXCEPTION 'tracker.if_modified_func() may only run as an AFTER trigger';
+  END IF;
+
+  -- get args
+
+  entity_pkey_name = TG_ARGV[0];
+
+  IF TG_ARGV[1] IS NOT NULL THEN
+    excluded_cols = TG_ARGV[1]::text[];
+  END IF;
+
+  -- fill row
+
+  event = ROW(
+    nextval('tracker.events_id_seq'),   -- id
+    TG_RELID,                           -- relation OID for much quicker searches
+    TG_TABLE_SCHEMA::text,              -- table_schema
+    TG_TABLE_NAME::text,                -- table_name
+    TG_OP::tracker.actions,             -- 'INSERT'/'UPDATE'/'DELETE'
+    NULL,                               -- primary key
+    NULL,                               -- row_data
+    NULL,                               -- changed_fields
+    clock_timestamp(),                  -- created_at
+    NULL                                -- checked_out_at
+  );
+
+  IF (TG_OP = 'UPDATE') THEN
+    h_old = hstore(OLD.*);
+    h_new = hstore(NEW.*);
+
+    event.entity_pkey              = h_old -> entity_pkey_name;
+    event.row_data          = h_old - excluded_cols;
+    event.changed_fields    = (h_new - event.row_data) - excluded_cols;
+    IF event.changed_fields = hstore('') THEN
+      -- All changed fields are ignored. Skip this update.
+      RETURN NULL;
+    END IF;
+  ELSIF (TG_OP = 'DELETE') THEN
+    h_old = hstore(OLD.*);
+    event.entity_pkey     = h_old -> entity_pkey_name;
+    event.row_data = h_old - excluded_cols;
+  ELSIF (TG_OP = 'INSERT') THEN
+    h_new              = hstore(NEW.*);
+    event.entity_pkey     = h_new -> entity_pkey_name;
+    event.row_data = h_new - excluded_cols;
+  END IF;
+  INSERT INTO tracker.events VALUES (event.*);
+  RETURN NULL;
+END;
+$body$
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog, public;
+
+/*
+
+Add tracking support to a table.
+
+Arguments:
+   target_table:     Table name, schema qualified if not on search_path
+   primary_key_name: Name of primary key column
+   ignored_cols:     Columns to exclude from update diffs, ignore updates that change only ignored cols.
+
+*/
+
+CREATE OR REPLACE FUNCTION tracker.track_table(target_table regclass, entity_pkey_name text, ignored_cols text[]) 
+  RETURNS void AS $body$
+DECLARE
+  _q_txt text;
+  _ignored_cols_snip text = '';
+BEGIN
+  IF array_length(ignored_cols,1) > 0 THEN
+    _ignored_cols_snip = ', ' || quote_literal(ignored_cols);
+  END IF;
+
+  EXECUTE 'DROP TRIGGER IF EXISTS track_trigger_row ON ' || quote_ident(target_table::TEXT);
+
+  _q_txt = 'CREATE TRIGGER track_trigger_row AFTER INSERT OR UPDATE OR DELETE ON ' || 
+              quote_ident(target_table::TEXT) || 
+              ' FOR EACH ROW EXECUTE PROCEDURE tracker.if_modified_func(' ||
+              quote_literal(entity_pkey_name) || 
+              _ignored_cols_snip || ');';
+  EXECUTE _q_txt;
+END;
+$body$
+language 'plpgsql';
+
+
+/*
+
+Add tracking support to the given table. No cols are ignored. (Shortcut)
+
+*/
+
+CREATE OR REPLACE FUNCTION tracker.track_table(target_table regclass) RETURNS void AS $body$
+SELECT tracker.track_table($1, 'id', ARRAY[]::text[]);
+$body$ LANGUAGE 'sql';
+
+COMMENT ON FUNCTION tracker.track_table(regclass) IS $body$
+$body$;
+
+
+--
+------------------------------------------------------------------------------------
+--
+
+-- SELECT tracker.track_table('public.posts');
+
+--SELECT tracker.track_table('public.users', 'id', array['created_at','updated_at']);
+-- -- SELECT tracker.track_table('public.users', ARRAY[]::text[]);
+-- -- SELECT tracker.track_table('public.users');
+--
+-- INSERT INTO users (email, created_at, updated_at) VALUES('me@mo' || (1000 * random())::integer , NOW(), NOW());
+-- UPDATE users SET username='mimi' WHERE email LIKE '%mo%';
+-- UPDATE users SET username='momo' WHERE email LIKE '%mo%';
+-- DELETE FROM users WHERE email LIKE '%mo%';
+--
+-- SELECT table_schema || '.' || table_name || '.' || entity_pkey, action, row_data, changed_fields, created_at FROM tracker.events;

data/lib/tracker.rb

@@ -0,0 +1,125 @@
+require_relative "tracker/migration"
+require_relative "tracker/registry"
+require_relative "tracker/advisory_lock"
+
+# module Tracker
+#   extend Registry
+#
+#   def self.on(*args, &block)
+#     Registry.on(*args, &block)
+#   end
+#
+#   def self.track(*args, &block)
+#     Registry.track(*args, &block)
+#   end
+#
+#   extend Migration
+#
+#   class Event < ActiveRecord::Base
+#     self.table_name = "tracker.events"
+#
+#     include AdvisoryLock
+#
+#     def self.track!(actions)
+#       where(id: actions.map(&:id)).update_all(checked_out_at: Time.now)
+#     end
+#   end
+#
+#   def self.check_out_events
+#     Event.exclusive do
+#       event = Event.where("checked_out_at IS NULL OR checked_out_at < ?", Time.now - 10.minutes).order(:id).first
+#       return [] unless event
+#
+#       if Registry.tracks?(event.table_name)
+#         events = Event.where(table_name: event.table_name, entity_pkey: event.entity_pkey).to_a
+#       else
+#         events = [ event ]
+#       end
+#
+#       Event.where(id: events.map(&:id)).update_all(checked_out_at: Time.now)
+#       events
+#     end
+#   end
+#
+#   def self.check_all!
+#     loop do
+#       events = check_out_events
+#       break if events.empty?
+#
+#       events.each do |event|
+#         publish_event(event)
+#       end
+#
+#       next if events.length == 1
+#
+#       # generate a table level event if necessary:
+#       #
+#       # A table level event is the last event, but only if the last event
+#       # is not a delete where the accompanying "create" is also in the
+#       # "events" array. In that case it would be the first entry
+#
+#       first_event = events.first
+#       last_event = events.last
+#       if first_event.action != "INSERT" && last_event.action != "DELETE"
+#         publish_event(last_event, event_name: last_event.table_name)
+#       end
+#
+#       destroyed = Event.where(id: events.map(&:id)).delete_all
+#       puts "destroyed #{destroyed} events"
+#     end
+#   end
+#
+#   def self.publish_event(event, event_name: nil)
+#     event_name ||= "#{event.table_name}.#{event.action.to_s.downcase}"
+#     Registry.publish! event_name, event.entity_pkey, event.row_data, event.changed_fields, event
+#   end
+# end
+#
+# Tracker.migrate!
+# Tracker.track_table! "public.posts"
+#
+# Tracker::Registry.reset!
+#
+# #
+# # Is called after a posts entry was created
+# Tracker.on "posts.insert" do |id, attrs|
+#   puts "--> created post ##{id} with attrs #{attrs.inspect}"
+# end
+#
+# #
+# # Is called after a posts entry was destroyed. attrs contains
+# # the previous attributes of the posts table.
+# Tracker.on "posts.delete" do |id, attrs|
+#   puts "--> deleted post ##{id} with attrs #{attrs.inspect}"
+# end
+#
+# #
+# # Is called whenever a post was changed. attrs contains the new
+# # set of attributes, changed_attrs contains the changed attributes.
+# Tracker.on "posts.update" do |id, attrs, changed_attrs|
+#   puts "--> updated post ##{id} with attrs #{attrs.inspect}\n  changed_attrs: #{changed_attrs.inspect}"
+# end
+#
+# post = Post.create! body: "bobo", author: User.first, group: User.first.wall
+# post.update! body: "yiha"
+# post.update! body: "banzai"
+# post.destroy
+#
+# Tracker.check_all!
+#
+# #
+# # Is called with the latest attrs for the "posts" entry.
+# Tracker.track "posts" do |id, attrs|
+#   puts "--> tracked post: id: #{id}, attrs: #{attrs}"
+# end
+#
+# post = Post.create! body: "bobo", author: User.first, group: User.first.wall
+# post.update! body: "yiha"
+# post.update! body: "banzai"
+# Tracker.check_all!
+#
+# post = Post.create! body: "bobo", author: User.first, group: User.first.wall
+# Tracker.check_all!
+#
+# post.destroy
+# Tracker.check_all!

data/spec/postqueue/concurrency_spec.rb

@@ -0,0 +1,77 @@
+require "spec_helper"
+
+describe "concurrency tests" do
+  # -- helper methods ---------------------------------------------------------
+
+  def processed_ids
+    File.read(LOG_FILE).split("\n").map(&:to_i)
+  end
+
+  def benchmark(msg, &block)
+    realtime = Benchmark.realtime(&block)
+    STDERR.puts "#{msg}: #{'%.3f secs' % realtime}"
+    realtime
+  end
+
+  LOG_FILE = "log/test-runner.log"
+
+  # Each runner writes the processed message into the LOG_FILE
+  def runner
+    ActiveRecord::Base.connection_pool.with_connection do |_conn|
+      log = File.open(LOG_FILE, "a")
+      queue = Postqueue.new
+      queue.on '*' do |_op, entity_ids|
+        sleep(0.0001); log.write "#{entity_ids.first}\n"
+      end
+      queue.process_until_empty
+      log.close
+    end
+  rescue => e
+    STDERR.puts "runner aborts: #{e}, from #{e.backtrace.first}"
+  end
+
+  def run_scenario(cnt, n_threads)
+    FileUtils.rm_rf LOG_FILE
+
+    queue = Postqueue.new do |queue|
+      # queue.default_batch_size = 10
+    end
+
+    benchmark "enqueuing #{cnt} ops" do
+      queue.enqueue op: "myop", entity_id: (1..cnt)
+    end
+
+    benchmark "processing #{cnt} ops with #{n_threads} threads" do
+      if n_threads == 0
+        runner
+      else
+        Array.new(n_threads) { Thread.new { runner } }.each(&:join)
+      end
+    end
+  end
+
+  # -- tests start here -------------------------------------------------------
+
+  it "runs faster with multiple runners", transactions: false do
+    # For small cnt values here the test below actually fails.
+    cnt = 1000
+
+    t0_runtime = run_scenario cnt, 0
+    expect(processed_ids).to contain_exactly(*(1..cnt).to_a)
+
+    t4_runtime = run_scenario cnt, 4
+    expect(processed_ids).to contain_exactly(*(1..cnt).to_a)
+    expect(t4_runtime).to be < t0_runtime * 0.8
+  end
+
+  it "enqueues many entries" do
+    cnt = 1000
+
+    queue = Postqueue.new do |queue|
+      # queue.default_batch_size = 10
+    end
+    benchmark "enqueuing #{cnt} ops" do
+      queue.enqueue op: "myop", entity_id: (1..cnt)
+    end
+  end
+end

data/spec/postqueue/enqueue_spec.rb

@@ -1,8 +1,9 @@
 require "spec_helper"
 
 describe "enqueuing" do
-  let(:queue) { Postqueue::Base.new }
-  let(:item)  { queue.items.first }
+  let(:queue) { Postqueue.new }
+  let(:items) { queue.item_class.all }
+  let(:item)  { queue.item_class.first }
 
   context "when enqueueing entries" do
     before do
@@ -20,39 +21,4 @@ describe "enqueuing" do
       expect(item.failed_attempts).to eq(0)
     end
   end
-
-  context "when enqueueing identical duplicate entries" do
-    before do
-      queue.enqueue op: "duplicate", entity_id: 12, duplicate: duplicate
-      queue.enqueue op: "duplicate", entity_id: 13, duplicate: duplicate
-      queue.enqueue op: "duplicate", entity_id: 12, duplicate: duplicate
-      queue.enqueue op: "duplicate", entity_id: 12, duplicate: duplicate
-      queue.enqueue op: "duplicate", entity_id: 12, duplicate: duplicate
-      queue.enqueue op: "no-duplicate", entity_id: 13, duplicate: duplicate
-    end
-
-    context "when duplicates are permitted" do
-      let(:duplicate) { true }
-
-      it "does not skip duplicates" do
-        expect(items.map(&:entity_id)).to eq([12, 13, 12, 12, 12, 13])
-      end
-    end
-
-    context "when duplicates are not permitted" do
-      let(:duplicate) { false }
-
-      it "skips later duplicates" do
-        expect(items.map(&:entity_id)).to eq([12, 13, 13])
-      end
-    end
-  end
-
-  context "when enqueueing many entries" do
-    it "adds all entries skipping duplicates" do
-      queue.enqueue op: "duplicate", entity_id: 12, duplicate: false
-      queue.enqueue op: "duplicate", entity_id: [13, 12, 12, 13, 14], duplicate: false
-      expect(items.map(&:entity_id)).to eq([12, 13, 14])
-    end
-  end
 end

data/spec/postqueue/idempotent_ops_spec.rb

@@ -0,0 +1,66 @@
+require "spec_helper"
+
+describe "idempotent operations" do
+  let(:queue) do
+    Postqueue.new do |queue|
+      queue.batch_sizes["batchable"] = 10
+      queue.idempotent_operation "idempotent"
+    end
+  end
+
+  let(:items) { queue.item_class.all }
+  let(:item)  { queue.item_class.first }
+
+  context "when enqueueing many entries" do
+    before do
+      queue.enqueue op: "idempotent", entity_id: 12
+      queue.enqueue op: "idempotent", entity_id: 13
+      queue.enqueue op: "idempotent", entity_id: 12
+      queue.enqueue op: "idempotent", entity_id: 12
+      queue.enqueue op: "idempotent", entity_id: 12
+      queue.enqueue op: "no-duplicate", entity_id: 14
+      queue.enqueue op: "no-duplicate", entity_id: 14
+    end
+
+    it "does not skip non-duplicates" do
+      entity_ids = items.select { |i| i.op == "no-duplicate" }.map(&:entity_id)
+      expect(entity_ids).to eq([14, 14])
+    end
+
+    it "skips duplicates" do
+      entity_ids = items.select { |i| i.op == "idempotent" }.map(&:entity_id)
+      expect(entity_ids).to eq([12, 13])
+    end
+  end
+
+  context "when enqueueing many entries" do
+    it "skips duplicates in entries" do
+      queue.enqueue op: "idempotent", entity_id: 12
+      queue.enqueue op: "idempotent", entity_id: [13, 12, 12, 13, 14]
+      queue.enqueue op: "idempotent", entity_id: 14
+      expect(items.map(&:entity_id)).to eq([12, 13, 14])
+    end
+  end
+
+  context "when processing entries" do
+    let(:callback_invocations) { @callback_invocations ||= [] }
+
+    before do
+      queue.enqueue op: "idempotent", entity_id: 12
+      queue.item_class.insert_item(op: "idempotent", entity_id: 12)
+
+      queue.on "idempotent" do |op, entity_ids|
+        callback_invocations << [ op, entity_ids ]
+      end
+      queue.process
+    end
+
+    it "runs the process callback only once" do
+      expect(callback_invocations.length).to eq(1)
+    end
+
+    it "removes all items" do
+      expect(items.count).to eq(0)
+    end
+  end
+end