metka 2.3.4 → 3.0.1

data/lib/generators/metka/sql_identifier.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Metka
+  module Generators
+    # Generator options land verbatim in the migration's SQL, so they are
+    # constrained to plain identifiers: a hostile-looking name fails closed
+    # instead of producing broken or surprising DDL.
+    module SqlIdentifier
+      IDENTIFIER = /\A[a-zA-Z_][a-zA-Z0-9_]*\z/
+
+      private
+
+      def validate_sql_identifiers!(names_by_option)
+        names_by_option.each do |option, names|
+          names.each do |name|
+            next if IDENTIFIER.match?(name)
+
+            raise Thor::Error, "#{option} #{name.inspect} must be a plain SQL identifier: " \
+              "letters, digits and underscores, not starting with a digit"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/generators/metka/strategies/index/index_generator.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+require_relative "../../sql_identifier"
+
+module Metka
+  module Generators
+    module Strategies
+      class IndexGenerator < ::Rails::Generators::Base # :nodoc:
+        include Rails::Generators::Migration
+        include Metka::Generators::SqlIdentifier
+
+        DEFAULT_SOURCE_COLUMNS = [ "tags" ].freeze
+
+        desc <<~LONGDESC
+          Generates a migration implementing the SQLite index strategy for
+          Metka: one (tag_name, record_id) side table per tagged column,
+          maintained by triggers, so tag queries become index seeks instead
+          of json_each table scans. Declare the tables on the model with the
+          index_tables option to route queries through them.
+
+          PostgreSQL needs no index strategy — GIN indexes on the array
+          columns already serve tag queries — so the generator produces
+          nothing there.
+
+          > $ rails g metka:strategies:index \
+          --source-table-name=NAME_OF_TABLE_WITH_TAGS \
+          --source-columns=NAME_OF_TAGGED_COLUMN_1 NAME_OF_TAGGED_COLUMN_2
+        LONGDESC
+
+        source_root File.expand_path("templates", __dir__)
+
+        class_option :source_table_name, type: :string, required: true,
+          desc: "Name of the table that has a column with tags"
+
+        class_option :source_columns, type: :array, default: DEFAULT_SOURCE_COLUMNS,
+          desc: "List of the tagged columns names"
+
+        def generate_migration
+          validate_sql_identifiers!(
+            "--source-table-name" => [ source_table_name ],
+            "--source-columns" => source_columns
+          )
+
+          unless sqlite?
+            say_status :skipped,
+              "the index strategy targets SQLite; PostgreSQL GIN indexes already serve tag queries",
+              :yellow
+            return
+          end
+
+          migration_template "migration.rb.erb", "db/migrate/#{migration_name}.rb"
+        end
+
+        no_tasks do
+          def sqlite?
+            ::ActiveRecord::Base.connection.adapter_name.match?(/sqlite/i)
+          end
+
+          def source_table_name
+            options[:source_table_name]
+          end
+
+          def source_columns
+            options[:source_columns]
+          end
+
+          def index_table_for(column)
+            "#{source_table_name}_#{column}_index"
+          end
+
+          def migration_name
+            "create_#{source_table_name}_#{source_columns.join('_and_')}_index_tables"
+          end
+        end
+
+        def self.next_migration_number(dir)
+          ::ActiveRecord::Generators::Base.next_migration_number(dir)
+        end
+      end
+    end
+  end
+end

data/lib/generators/metka/strategies/index/templates/migration.rb.erb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+class <%= @migration_class_name %> < ActiveRecord::Migration<%= ActiveRecord::VERSION::MAJOR < 5 ? '' : '[5.0]' %>
+  # One (tag_name, record_id) side table per tagged column, so SQLite tag
+  # queries become index seeks instead of json_each table scans. The WITHOUT
+  # ROWID primary key (tag_name, record_id) is itself the covering index —
+  # no separate index needed — and doubles as the dedup guard: a tag
+  # duplicated inside one row's array stores a single pair, which is all
+  # membership queries need.
+  #
+  # Per-row triggers keep the pairs in step with every INSERT, UPDATE and
+  # DELETE, multi-row statements included. Deletes seek through the primary
+  # key (tag_name from OLD's array, then record_id), so no extra index on
+  # record_id is required. Writes that bypass the triggers (restoring from a
+  # dump) require reseeding the table by hand.
+  #
+  # Declare the tables on the model to route queries through them:
+  #
+  #   include Metka::Model(
+  #     columns: %w[<%= source_columns.join(' ') %>],
+  #     index_tables: {
+<% source_columns.each do |column| -%>
+  #       "<%= column %>" => "<%= index_table_for(column) %>",
+<% end -%>
+  #     }
+  #   )
+  def up
+<% source_columns.each do |column| -%>
+    execute <<-SQL
+    CREATE TABLE <%= index_table_for(column) %> (
+      tag_name varchar NOT NULL,
+      record_id bigint NOT NULL,
+      PRIMARY KEY (tag_name, record_id)
+    ) WITHOUT ROWID;
+    SQL
+
+    execute <<-SQL
+    INSERT OR IGNORE INTO <%= index_table_for(column) %> (tag_name, record_id)
+      SELECT value, <%= source_table_name %>.id
+      FROM <%= source_table_name %>, json_each(<%= source_table_name %>.<%= column %>);
+    SQL
+
+    execute <<-SQL
+    CREATE TRIGGER metka_idx_ins_on_<%= source_table_name %>_<%= column %>
+    AFTER INSERT ON <%= source_table_name %>
+    FOR EACH ROW
+    BEGIN
+      INSERT OR IGNORE INTO <%= index_table_for(column) %> (tag_name, record_id)
+      SELECT value, NEW.id FROM json_each(NEW.<%= column %>);
+    END;
+    SQL
+
+    execute <<-SQL
+    CREATE TRIGGER metka_idx_upd_on_<%= source_table_name %>_<%= column %>
+    AFTER UPDATE OF <%= column %> ON <%= source_table_name %>
+    FOR EACH ROW
+    BEGIN
+      DELETE FROM <%= index_table_for(column) %>
+      WHERE tag_name IN (SELECT value FROM json_each(OLD.<%= column %>))
+        AND record_id = OLD.id;
+
+      INSERT OR IGNORE INTO <%= index_table_for(column) %> (tag_name, record_id)
+      SELECT value, NEW.id FROM json_each(NEW.<%= column %>);
+    END;
+    SQL
+
+    execute <<-SQL
+    CREATE TRIGGER metka_idx_del_on_<%= source_table_name %>_<%= column %>
+    AFTER DELETE ON <%= source_table_name %>
+    FOR EACH ROW
+    BEGIN
+      DELETE FROM <%= index_table_for(column) %>
+      WHERE tag_name IN (SELECT value FROM json_each(OLD.<%= column %>))
+        AND record_id = OLD.id;
+    END;
+    SQL
+
+<% end -%>
+  end
+
+  def down
+<% source_columns.each do |column| -%>
+    execute "DROP TRIGGER IF EXISTS metka_idx_ins_on_<%= source_table_name %>_<%= column %>;"
+    execute "DROP TRIGGER IF EXISTS metka_idx_upd_on_<%= source_table_name %>_<%= column %>;"
+    execute "DROP TRIGGER IF EXISTS metka_idx_del_on_<%= source_table_name %>_<%= column %>;"
+    execute "DROP TABLE IF EXISTS <%= index_table_for(column) %>;"
+<% end -%>
+  end
+end

data/lib/generators/metka/strategies/table/table_generator.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+require_relative "../../sql_identifier"
+
+module Metka
+  module Generators
+    module Strategies
+      class TableGenerator < ::Rails::Generators::Base # :nodoc:
+        include Rails::Generators::Migration
+        include Metka::Generators::SqlIdentifier
+
+        DEFAULT_SOURCE_COLUMNS = [ "tags" ].freeze
+
+        desc <<~LONGDESC
+          Generates migration to implement table strategy for Metka
+
+          > $ rails g metka:strategies:table \
+          --source-table-name=NAME_OF_TABLE_WITH_TAGS \
+          --source-columns=NAME_OF_TAGGED_COLUMN_1 NAME_OF_TAGGED_COLUMN_2 \
+          --table-name=NAME_OF_TABLE
+        LONGDESC
+
+        source_root File.expand_path("templates", __dir__)
+
+        class_option :source_table_name, type: :string, required: true,
+          desc: "Name of the table that has a column with tags"
+
+        class_option :source_columns, type: :array, default: DEFAULT_SOURCE_COLUMNS,
+          desc: "List of the tagged columns names"
+
+        class_option :table_name, type: :string,
+          desc: "Custom name for the resulting table"
+
+        def generate_migration
+          validate_sql_identifiers!(
+            "--source-table-name" => [ source_table_name ],
+            "--source-columns" => source_columns,
+            "--table-name" => Array(options[:table_name])
+          )
+
+          migration_template migration_template_file, "db/migrate/#{migration_name}.rb"
+        end
+
+        no_tasks do
+          # The migration is written for the database the app is connected to
+          # at generation time: transition-table triggers for PostgreSQL,
+          # per-row json_each triggers for SQLite.
+          def migration_template_file
+            if ::ActiveRecord::Base.connection.adapter_name.match?(/sqlite/i)
+              "migration.sqlite.rb.erb"
+            else
+              "migration.rb.erb"
+            end
+          end
+
+          def source_table_name
+            options[:source_table_name]
+          end
+
+          def source_columns
+            options[:source_columns]
+          end
+
+          def source_columns_names
+            source_columns.join("_and_")
+          end
+
+          def table_name
+            return options[:table_name] if options[:table_name]
+
+            "#{source_table_name}_#{source_columns_names}_cloud"
+          end
+
+          def migration_name
+            "create_#{table_name}_table"
+          end
+
+          def migration_class_name
+            migration_name.classify
+          end
+        end
+
+        def self.next_migration_number(dir)
+          ::ActiveRecord::Generators::Base.next_migration_number(dir)
+        end
+      end
+    end
+  end
+end

data/lib/generators/metka/strategies/table/templates/migration.rb.erb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+class <%= @migration_class_name %> < ActiveRecord::Migration<%= ActiveRecord::VERSION::MAJOR < 5 ? '' : '[5.0]' %>
+  def up
+    execute <<-SQL
+    -- A real table maintained by per-tag deltas: statement-level triggers read
+    -- the transition tables and upsert only the tags the statement touched, so
+    -- maintenance cost is O(tags touched) instead of a full recompute. Writes
+    -- that bypass these triggers (TRUNCATE, restoring from a dump) require
+    -- reseeding the table by hand.
+
+    -- Block writes (reads stay unblocked) until the triggers exist: a write
+    -- committing between the seed's snapshot and CREATE TRIGGER would be seen
+    -- by neither and drift the counts. CREATE TRIGGER takes this same lock
+    -- level anyway; this only takes it before the seed instead of after.
+    LOCK TABLE <%= source_table_name %> IN SHARE ROW EXCLUSIVE MODE;
+
+    CREATE TABLE <%= table_name %> (
+      tag_name varchar PRIMARY KEY,
+      taggings_count bigint NOT NULL
+    );
+
+    INSERT INTO <%= table_name %> (tag_name, taggings_count)
+      SELECT
+        tag_name,
+        COUNT(*) AS taggings_count
+      FROM (
+        SELECT UNNEST
+          (<%= source_columns.join(' || ') %>) AS tag_name
+        FROM
+          <%= source_table_name %>
+      ) subquery
+      GROUP BY
+        tag_name;
+
+    -- UNNEST of a NULL array yields no rows and array concatenation treats a
+    -- NULL operand as empty, so NULL tagged columns need no explicit guards.
+    -- One function per operation: a transition table is only registered for
+    -- its own trigger, so the other operations' functions would fail to parse.
+    CREATE OR REPLACE FUNCTION metka_ins_<%= table_name %>() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        INSERT INTO <%= table_name %> (tag_name, taggings_count)
+        SELECT tag_name, COUNT(*)
+        FROM (
+          SELECT UNNEST (<%= source_columns.join(' || ') %>) AS tag_name
+          FROM new_rows
+        ) subquery
+        GROUP BY tag_name
+        ON CONFLICT (tag_name)
+        DO UPDATE SET taggings_count = <%= table_name %>.taggings_count + EXCLUDED.taggings_count;
+        RETURN NULL;
+      END $$;
+
+    CREATE OR REPLACE FUNCTION metka_upd_<%= table_name %>() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        WITH deltas AS (
+          SELECT tag_name, SUM(d) AS delta
+          FROM (
+            SELECT UNNEST (<%= source_columns.join(' || ') %>) AS tag_name, 1 AS d FROM new_rows
+            UNION ALL
+            SELECT UNNEST (<%= source_columns.join(' || ') %>) AS tag_name, -1 AS d FROM old_rows
+          ) changes
+          GROUP BY tag_name
+          HAVING SUM(d) <> 0
+        )
+        INSERT INTO <%= table_name %> (tag_name, taggings_count)
+        SELECT tag_name, delta FROM deltas
+        ON CONFLICT (tag_name)
+        DO UPDATE SET taggings_count = <%= table_name %>.taggings_count + EXCLUDED.taggings_count;
+
+        DELETE FROM <%= table_name %> WHERE taggings_count <= 0;
+        RETURN NULL;
+      END $$;
+
+    CREATE OR REPLACE FUNCTION metka_del_<%= table_name %>() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        UPDATE <%= table_name %>
+        SET taggings_count = <%= table_name %>.taggings_count - removed.taggings_count
+        FROM (
+          SELECT tag_name, COUNT(*) AS taggings_count
+          FROM (
+            SELECT UNNEST (<%= source_columns.join(' || ') %>) AS tag_name
+            FROM old_rows
+          ) subquery
+          GROUP BY tag_name
+        ) removed
+        WHERE <%= table_name %>.tag_name = removed.tag_name;
+
+        DELETE FROM <%= table_name %> WHERE taggings_count <= 0;
+        RETURN NULL;
+      END $$;
+
+    -- The ins/upd/del discriminator sits at the front of the trigger names so
+    -- they stay distinct even when PostgreSQL truncates identifiers to 63
+    -- characters. One trigger per operation: a trigger with transition tables
+    -- must be declared for exactly one event.
+    CREATE TRIGGER metka_ins_on_<%= source_table_name %>_<%= source_columns_names %>
+    AFTER INSERT ON <%= source_table_name %>
+    REFERENCING NEW TABLE AS new_rows
+    FOR EACH STATEMENT
+    EXECUTE PROCEDURE metka_ins_<%= table_name %>();
+
+    CREATE TRIGGER metka_upd_on_<%= source_table_name %>_<%= source_columns_names %>
+    AFTER UPDATE ON <%= source_table_name %>
+    REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
+    FOR EACH STATEMENT
+    EXECUTE PROCEDURE metka_upd_<%= table_name %>();
+
+    CREATE TRIGGER metka_del_on_<%= source_table_name %>_<%= source_columns_names %>
+    AFTER DELETE ON <%= source_table_name %>
+    REFERENCING OLD TABLE AS old_rows
+    FOR EACH STATEMENT
+    EXECUTE PROCEDURE metka_del_<%= table_name %>();
+    SQL
+  end
+
+  def down
+    execute <<-SQL
+      DROP TRIGGER IF EXISTS metka_ins_on_<%= source_table_name %>_<%= source_columns_names %> ON <%= source_table_name %>;
+      DROP TRIGGER IF EXISTS metka_upd_on_<%= source_table_name %>_<%= source_columns_names %> ON <%= source_table_name %>;
+      DROP TRIGGER IF EXISTS metka_del_on_<%= source_table_name %>_<%= source_columns_names %> ON <%= source_table_name %>;
+      DROP FUNCTION IF EXISTS metka_ins_<%= table_name %>;
+      DROP FUNCTION IF EXISTS metka_upd_<%= table_name %>;
+      DROP FUNCTION IF EXISTS metka_del_<%= table_name %>;
+      DROP TABLE IF EXISTS <%= table_name %>;
+    SQL
+  end
+end

data/lib/generators/metka/strategies/table/templates/migration.sqlite.rb.erb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+class <%= @migration_class_name %> < ActiveRecord::Migration<%= ActiveRecord::VERSION::MAJOR < 5 ? '' : '[5.0]' %>
+  # A real table maintained by per-tag deltas. SQLite has no statement-level
+  # triggers or transition tables, so per-row triggers read NEW/OLD
+  # through json_each and apply the deltas one row at a time; multi-row
+  # statements (insert_all, update_all, delete_all) still keep exact counts
+  # because the trigger fires once per affected row. SQLite allows one writer
+  # per database and DDL is transactional, so nothing can write between the
+  # seed and CREATE TRIGGER and no explicit lock is needed. Writes that bypass
+  # the triggers (restoring from a dump) require reseeding the table by hand.
+  #
+  # Requires SQLite >= 3.35 for ON CONFLICT inside a trigger body. The
+  # WHERE true in the upsert disambiguates ON CONFLICT from a join clause in
+  # the INSERT ... SELECT form — a documented SQLite parser requirement.
+  # json_each(NULL) yields no rows, so NULL tagged columns need no guards, and
+  # a tag duplicated inside one row's array upserts once per occurrence —
+  # counted exactly as PostgreSQL's UNNEST-based triggers count it.
+  def up
+    execute <<-SQL
+    CREATE TABLE <%= table_name %> (
+      tag_name varchar PRIMARY KEY,
+      taggings_count bigint NOT NULL
+    );
+    SQL
+
+    execute <<-SQL
+    INSERT INTO <%= table_name %> (tag_name, taggings_count)
+      SELECT tag_name, COUNT(*)
+      FROM (
+        <%= source_columns.map { |column| "SELECT value AS tag_name FROM #{source_table_name}, json_each(#{source_table_name}.#{column})" }.join("\n        UNION ALL\n        ") %>
+      )
+      GROUP BY tag_name;
+    SQL
+
+    execute <<-SQL
+    CREATE TRIGGER metka_ins_on_<%= source_table_name %>_<%= source_columns_names %>
+    AFTER INSERT ON <%= source_table_name %>
+    FOR EACH ROW
+    BEGIN
+<% source_columns.each do |column| -%>
+      INSERT INTO <%= table_name %> (tag_name, taggings_count)
+      SELECT value, 1 FROM json_each(NEW.<%= column %>) WHERE true
+      ON CONFLICT (tag_name)
+      DO UPDATE SET taggings_count = taggings_count + 1;
+<% end -%>
+    END;
+    SQL
+
+    execute <<-SQL
+    CREATE TRIGGER metka_upd_on_<%= source_table_name %>_<%= source_columns_names %>
+    AFTER UPDATE OF <%= source_columns.join(", ") %> ON <%= source_table_name %>
+    FOR EACH ROW
+    BEGIN
+<% source_columns.each do |column| -%>
+      INSERT INTO <%= table_name %> (tag_name, taggings_count)
+      SELECT value, 1 FROM json_each(NEW.<%= column %>) WHERE true
+      ON CONFLICT (tag_name)
+      DO UPDATE SET taggings_count = taggings_count + 1;
+<% end -%>
+<% source_columns.each do |column| -%>
+      UPDATE <%= table_name %>
+      SET taggings_count = taggings_count -
+        (SELECT COUNT(*) FROM json_each(OLD.<%= column %>) WHERE value = tag_name)
+      WHERE tag_name IN (SELECT value FROM json_each(OLD.<%= column %>));
+<% end -%>
+      DELETE FROM <%= table_name %> WHERE taggings_count <= 0;
+    END;
+    SQL
+
+    execute <<-SQL
+    CREATE TRIGGER metka_del_on_<%= source_table_name %>_<%= source_columns_names %>
+    AFTER DELETE ON <%= source_table_name %>
+    FOR EACH ROW
+    BEGIN
+<% source_columns.each do |column| -%>
+      UPDATE <%= table_name %>
+      SET taggings_count = taggings_count -
+        (SELECT COUNT(*) FROM json_each(OLD.<%= column %>) WHERE value = tag_name)
+      WHERE tag_name IN (SELECT value FROM json_each(OLD.<%= column %>));
+<% end -%>
+      DELETE FROM <%= table_name %> WHERE taggings_count <= 0;
+    END;
+    SQL
+  end
+
+  def down
+    execute "DROP TRIGGER IF EXISTS metka_ins_on_<%= source_table_name %>_<%= source_columns_names %>;"
+    execute "DROP TRIGGER IF EXISTS metka_upd_on_<%= source_table_name %>_<%= source_columns_names %>;"
+    execute "DROP TRIGGER IF EXISTS metka_del_on_<%= source_table_name %>_<%= source_columns_names %>;"
+    execute "DROP TABLE IF EXISTS <%= table_name %>;"
+  end
+end

data/lib/metka/generic_parser.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'singleton'
+require "singleton"
 
 module Metka
   ##
@@ -13,19 +13,24 @@ module Metka
     include Singleton
 
     def initialize
-      @single_quote_pattern ||= {}
-      @double_quote_pattern ||= {}
+      @separator = {}
+      @single_quote_pattern = {}
+      @double_quote_pattern = {}
     end
 
     def call(value)
+      # A TagList is this parser's own output, so it is already split,
+      # stripped, and de-duplicated — hand it back rather than rebuilding it.
+      return value if value.is_a?(TagList)
+
       TagList.new.tap do |tag_list|
         case value
         when String
-          value = value.to_s.dup
-          gsub_quote_pattern!(tag_list, value, double_quote_pattern)
-          gsub_quote_pattern!(tag_list, value, single_quote_pattern)
+          unquoted = value.dup
 
-          tag_list.merge value.split(Regexp.new(delimiter)).map(&:strip).reject(&:empty?)
+          tag_list.merge extract_quoted!(unquoted, double_quote_pattern)
+          tag_list.merge extract_quoted!(unquoted, single_quote_pattern)
+          tag_list.merge unquoted.split(separator).map(&:strip).reject(&:empty?)
         when Enumerable
           tag_list.merge value.reject(&:empty?)
         end
@@ -34,23 +39,32 @@ module Metka
 
     private
 
-    def gsub_quote_pattern!(tag_list, value, pattern)
-      value.gsub!(pattern) {
-        tag_list.add(Regexp.last_match[2])
-        ''
-      }
+    # Returns the quoted tags and strips them out of +text+, which is left
+    # holding only the unquoted remainder for the delimiter split to handle.
+    def extract_quoted!(text, pattern)
+      tags = []
+      text.gsub!(pattern) { tags << Regexp.last_match[:tag]; "" }
+      tags
     end
 
     def delimiter
       Metka.delimiter
     end
 
+    # The delimiter is a literal separator, so it is escaped before going
+    # anywhere near a Regexp. Without this a delimiter of "|" becomes an empty
+    # alternation matching between every character, and "." matches every
+    # character — both silently shredding the input instead of splitting it.
+    def separator
+      @separator[delimiter] ||= Regexp.new(Regexp.escape(delimiter))
+    end
+
     def single_quote_pattern
-      @single_quote_pattern[delimiter] ||= /(\A|#{delimiter})\s*'(.*?)'\s*(?=#{delimiter}\s*|\z)/
+      @single_quote_pattern[delimiter] ||= /(?:\A|#{Regexp.escape(delimiter)})\s*'(?<tag>.*?)'\s*(?=#{Regexp.escape(delimiter)}\s*|\z)/
     end
 
     def double_quote_pattern
-      @double_quote_pattern[delimiter] ||= /(\A|#{delimiter})\s*"(.*?)"\s*(?=#{delimiter}\s*|\z)/
+      @double_quote_pattern[delimiter] ||= /(?:\A|#{Regexp.escape(delimiter)})\s*"(?<tag>.*?)"\s*(?=#{Regexp.escape(delimiter)}\s*|\z)/
     end
   end
 end