directory_diff 0.4.14 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c5895d2d77767028f8894d7b9148b487fc0f7a3d
-  data.tar.gz: ce8f0d7b29b8028735c46959f88a39e628a786b6
+SHA256:
+  metadata.gz: d8ee4b73f492c39e557743dfb1fe4c8b84cd10dc4807f11d4606dad2cecf3327
+  data.tar.gz: 918743f81b8d6bc18b5a7ba5d8722efc67a7af35fa718e7095b63b5ed6c91961
 SHA512:
-  metadata.gz: 71c87ae049b8362daa88922991a8cec444713baa09f1de7a8ffd1a29aec2464e18828bde4d965871a457d95fba0e4d2845333f9c80f45d862331fd60032edf78
-  data.tar.gz: d22c8496602c2ba38f8e48cda9738b679093d81fcb22fcbdadde91b9a01c689d08c0a6c597f2dab523fb1686edf1f05aceb2657728c7c0f26d8f171aa20a3ccf
+  metadata.gz: 4cf77d75a44f310640a5ce99e4b569a5a7e82e046647b6c70fabf7407c27e460dea47003d376935132bbabe2ca8b088dea9052c29c39f90b02e6dcd145d88714
+  data.tar.gz: 8590c90ab336c515bb9ab7633caf380f1ca9ed9dd7bf8ed30e8a72e85cc1c8bd857cd8ec6a9b1f2823129e5663242fccb7692e0171bb2abd29fcd30a58856abb

data/directory_diff.gemspec

@@ -21,7 +21,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord", ">= 5.1"
   spec.add_dependency "pg", "~> 1.1.3"
-  spec.add_dependency "activerecord_pg_stuff", "~> 0.2.1"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/directory_diff/transform.rb

@@ -1,5 +1,4 @@
 require_relative "transformer/in_memory"
-require_relative "transformer/temp_table"
 
 module DirectoryDiff
   class Transform
@@ -10,21 +9,7 @@ module DirectoryDiff
     end
 
     def into(new_directory, options = {})
-      processor_class = processor_for(options[:processor])
-      processor_class.new(current_directory).into(new_directory, options)
-    end
-
-    private
-
-    def processor_for(processor)
-      case processor
-      when nil, :in_memory
-        Transformer::InMemory
-      when :temp_table
-        Transformer::TempTable
-      else
-        raise ArgumentError, "unsupported processor #{processor.inspect}"
-      end
+      Transformer::InMemory.new(current_directory).into(new_directory, options)
     end
   end
 end

data/lib/directory_diff/transformer/in_memory.rb

@@ -1,6 +1,6 @@
 module DirectoryDiff
   module Transformer
-    class InMemory 
+    class InMemory
       attr_reader :current_directory, :new_directory
       attr_reader :transforms, :transforms_index
       attr_reader :options
@@ -59,7 +59,7 @@ module DirectoryDiff
           # the new record appears to be the same as the current record
           if assistant_emails.empty?
             original_assistant_value = nil
-            new_employee[3] = old_employee&.fetch(3) 
+            new_employee[3] = old_employee&.fetch(3)
           else
             original_assistant_value = new_employee[3]
           end
@@ -77,7 +77,7 @@ module DirectoryDiff
 
           if old_employee.nil?
             add_transform(:insert, new_employee)
-          elsif new_employee[0, 4] == old_employee[0, 4]
+          elsif new_employee[0, 5] == old_employee[0, 5]
             # restore assistant value after cleanup like missing assistants and own email
             new_employee[3] = original_assistant_value
             # restore phone number value

data/lib/directory_diff/version.rb

@@ -1,3 +1,3 @@
 module DirectoryDiff
-  VERSION = "0.4.14"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: directory_diff
 version: !ruby/object:Gem::Version
-  version: 0.4.14
+  version: 0.5.0
 platform: ruby
 authors:
 - Kamal Mahyuddin
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-10-16 00:00:00.000000000 Z
+date: 2023-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,20 +38,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.3
-- !ruby/object:Gem::Dependency
-  name: activerecord_pg_stuff
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.2.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -149,13 +135,12 @@ files:
 - lib/directory_diff.rb
 - lib/directory_diff/transform.rb
 - lib/directory_diff/transformer/in_memory.rb
-- lib/directory_diff/transformer/temp_table.rb
 - lib/directory_diff/version.rb
 homepage: https://github.com/envoy/directory_diff
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -170,9 +155,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.3.12
+signing_key:
 specification_version: 4
 summary: Envoy employee directory diffing.
 test_files: []

data/lib/directory_diff/transformer/temp_table.rb

@@ -1,385 +0,0 @@
-# frozen_string_literal: true
-
-require "activerecord_pg_stuff"
-
-module DirectoryDiff
-  module Transformer
-    class TempTable
-      attr_reader :current_directory, :operations
-
-      # @params current_directory a relation that filters out only the records
-      #                           that represent the current directory. This is
-      #                           mostly likely an Employee relation. This
-      #                           relation will be pulled into a temporary table.
-      def initialize(current_directory)
-        @current_directory = current_directory
-        @operations = []
-      end
-
-      # @param new_directory a table containing only the new records to compare
-      #                      against, most likely a temp table.
-      def into(new_directory, options = {})
-        current_directory_temp_table do |temp_current_directory|
-          new_directory_temp_table(new_directory) do |deduped_csv|
-            # Get Arel tables for referencing fields, table names
-            employees = temp_current_directory.table
-            csv = deduped_csv.table
-
-            # Reusable Arel predicates
-            csv_employee_join = csv[:email].eq(employees[:email])
-            attributes_unchanged = employees[:name].eq(csv[:name])
-                                    .and(
-                                      employees[:phone_number].eq(csv[:phone_number])
-                                        .or(csv[:phone_number].eq(""))
-                                        # ☝🏽 Comparing to an empty string because we cast
-                                        # phone number to an empty string. The reason is
-                                        # comparing NULL = NULL is always false in SQL
-                                    )
-                                    .and(
-                                      employees[:assistants].eq(csv[:assistants])
-                                        .or(csv[:assistants].eq("{}"))
-                                    )
-
-            # Creates joins between the temp table and the csv table and
-            # vice versa
-            # Cribbed from https://gist.github.com/mildmojo/3724189
-            csv_to_employees = csv.join(employees, Arel::Nodes::OuterJoin)
-                                .on(csv_employee_join)
-                                .join_sources
-            employees_to_csv = employees.join(csv, Arel::Nodes::OuterJoin)
-                                .on(csv_employee_join)
-                                .join_sources
-
-            # Representation of the joined csv-employees, with csv on the left
-            csv_records = deduped_csv.joins(csv_to_employees).order('row_number asc')
-            # Representation of the joined employees-csv, with employees on the
-            # left
-            employee_records = temp_current_directory.joins(employees_to_csv)
-
-            connection.execute(SQL.cleanup_sql(csv.name))
-
-            csv_fields = %I[name email phone_number assistants extra].map { |c| csv[c] }
-            emp_fields = %I[name email phone_number assistants].map { |c| employees[c] }
-
-            # new records are records in the new directory that don't exist in
-            # the current directory
-            new_records = csv_records
-                            .select("'insert'::varchar operation, row_number")
-                            .select(csv_fields)
-                            .where({ employees.name => { email: nil } })
-            # deleted records are records in the current directory that don't
-            # exist in the new directory
-            deleted_records = employee_records
-                                .select("'delete'::varchar operation, row_number")
-                                .select(emp_fields)
-                                .select("null extra")
-                                .where({ csv.name => { email: nil } })
-            # changed records are records that have difference in name, phone
-            # number and/or assistants
-            changed_records = csv_records
-                                .select("'update'::varchar operation, row_number")
-                                .select(csv_fields)
-                                .where.not(attributes_unchanged)
-            # unchanged records are records that are exactly the same in both
-            # directories (without considering the extra field)
-            unchanged_records = csv_records
-                                  .select("'noop'::varchar operation, row_number")
-                                  .select(csv_fields)
-                                  .where(attributes_unchanged)
-
-            # create temp table for holding operations
-            temp_table(new_records.to_sql) do |operations|
-              insert_into_operations(operations, deleted_records.to_sql)
-              insert_into_operations(operations, changed_records.to_sql)
-              if options[:skip_noop] != true
-                insert_into_operations(operations, unchanged_records.to_sql)
-              end
-
-              operations.order(:row_number).each do |operation|
-                add_operation(operation)
-              end
-            end
-          end
-        end
-
-        prioritize_assistants(operations)
-      end
-
-      private
-
-      def current_directory_temp_table(&block)
-        # outer temp table is required so that the projection does not run into
-        # ambiguous column issues
-        temp_table(current_directory) do |rel|
-          temp_table(rel.select(SQL.current_directory_projection), &block)
-        end
-      end
-
-      def new_directory_temp_table(source, &block)
-        convert_to_relation(source) do |relation|
-          relation = relation.select("*")
-            .from(Arel.sql("(#{SQL.latest_unique_sql(relation.table.name)}) as t"))
-            .order("row_number")
-
-          temp_table(relation, &block)
-        end
-      end
-
-      def convert_to_relation(source, &block)
-        return block.call(source) if source.is_a?(ActiveRecord::Relation)
-
-        temp_table do |relation|
-          table_name = relation.table.name
-          connection.change_table(table_name) do |t|
-            t.column :name, :string
-            t.column :email, :string
-            t.column :phone_number, :string
-            t.column :assistants, :string
-            t.column :extra, :string
-          end
-          insert_into_csv_table(table_name, source)
-          block.call(relation)
-        end
-      end
-
-      # TODO chunk this into batch sizes
-      def insert_into_csv_table(table_name, records)
-        return if records.empty?
-
-        values = records.map do |row|
-          (name, email, phone_number, assistants, extra) = row
-          columns = [
-            connection.quote(name),
-            connection.quote(email),
-            connection.quote(phone_number),
-            connection.quote(assistants),
-            connection.quote(extra)
-          ]
-          "(#{columns.join(", ")})"
-        end
-
-        connection.execute(SQL.insert_into_temp_csv_table(table_name, values))
-      end
-
-      def temp_table(source = nil, &block)
-        return source.temporary_table(&block) if source.is_a?(ActiveRecord::Relation)
-
-        create_temp_table(source) do |name|
-          klass = current_directory.klass
-          dec = ActiveRecordPgStuff::Relation::TemporaryTable::Decorator.new(klass, name)
-          if activerecord52?
-            rel = ActiveRecord::Relation.new(dec)
-          else
-            rel = ActiveRecord::Relation.new(
-              dec,
-              dec.arel_table,
-              dec.predicate_builder,
-              {}
-            )
-          end
-          rel.readonly!
-          block.call(rel)
-        end
-      end
-
-      def create_temp_table(initial_sql=nil)
-        table_name = "temporary_#{(Time.now.to_f * 1000).to_i}"
-
-        if initial_sql
-          connection.with_temporary_table(table_name, initial_sql) do |name|
-            yield name
-          end
-        else
-          connection.transaction do
-            begin
-              connection.create_table(table_name, temporary: true)
-              yield table_name
-            ensure
-              connection.drop_table(table_name)
-            end
-          end
-        end
-      end
-
-      def insert_into_operations(relation, sql)
-        connection.execute(SQL.insert_into_operations(relation.table.name, sql))
-      end
-
-      def add_operation(operation)
-        op = [
-          operation.operation.to_sym,
-          operation.name,
-          operation.email,
-          operation.phone_number.presence,
-          serialize_pg_array(operation.assistants)
-        ]
-        op << operation.extra unless operation[:extra].nil?
-        operations << op
-      end
-
-      def serialize_pg_array(pg_array)
-        return if pg_array.nil?
-        pg_array = pg_array[1...-1] # remove the curly braces
-        pg_array.presence
-      end
-
-      def prioritize_assistants(operations)
-        prioritized_operations = []
-        operations.each do |operation|
-          process_operation(operation, operations, prioritized_operations, Set.new)
-        end
-        prioritized_operations
-      end
-
-      def process_operation(operation, operations, prioritized_operations, tail)
-        (_, _, email, _, assistants) = operation
-        return if prioritized_operations.find { |_, _, e| e == email }
-
-        (assistants || '').split(',').each do |assistant_email|
-          next if tail.include?(assistant_email)
-          assistant_operation = operations.find { |_, _, email| email == assistant_email }
-          process_operation(
-            assistant_operation,
-            operations,
-            prioritized_operations,
-            tail.add(email)
-          )
-        end
-
-        prioritized_operations << operation
-      end
-
-      def connection
-        current_directory.connection
-      end
-
-      def activerecord52?
-        ActiveRecord.gem_version >= Gem::Version.new("5.2.x")
-      end
-    end
-
-    module SQL
-      # Cleanup some bad records
-      # 1. Assistant email is set on an employee, but no assistant record
-      #    in csv. Remove the assistant email.
-      # 2. Assistant email is employee's own email. Remove the assistant
-      #    email.
-      # TODO move this into the temp table creation above
-      # https://www.db-fiddle.com/f/gxg6qABP1LygYvvgRvyH2N/1
-      def self.cleanup_sql(table_name)
-        <<-SQL
-          with
-            unnested_assistants as
-            (
-              select
-                email,
-                name,
-                unnest(assistants) assistant
-              from #{table_name}
-            ),
-            own_email_removed as
-            (
-              select
-                a.*
-              from unnested_assistants a
-              where a.email != a.assistant
-            ),
-            missing_assistants_removed as
-            (
-              select
-                a.*
-              from own_email_removed a
-              left outer join #{table_name} b on a.assistant = b.email
-              where
-                (a.assistant is null and b.email is null)
-                or (a.assistant is not null and b.email is not null)
-            ),
-            only_valid_assistants as
-            (
-              select
-                a.email,
-                a.name,
-                array_remove(
-                  array_agg(b.assistant),
-                  null
-                ) assistants
-              from #{table_name} a
-              left outer join missing_assistants_removed b
-              using (email)
-              group by
-                a.email, a.name
-            )
-          update #{table_name}
-          set assistants = only_valid_assistants.assistants
-          from only_valid_assistants
-          where #{table_name}.email = only_valid_assistants.email
-        SQL
-      end
-
-      # Remove dupe email rows, keeping the last one
-      def self.latest_unique_sql(table_name)
-        <<-SQL
-          SELECT
-            DISTINCT ON (lower(email)) name,
-            lower(email) email,
-            coalesce(phone_number, '') phone_number,
-            array_remove(
-              regexp_split_to_array(
-                coalesce(assistants, ''),
-                '\s*,\s*'
-              )::varchar[],
-              ''
-            ) assistants,
-            extra,
-            ROW_NUMBER () OVER ()
-          FROM
-            #{table_name}
-          ORDER BY
-            lower(email),
-            row_number desc
-        SQL
-      end
-
-      def self.current_directory_projection
-        <<-SQL
-          name,
-          lower(email) email,
-          coalesce(phone_number, '') phone_number,
-          array_remove(
-            regexp_split_to_array(
-              coalesce(assistants, ''),
-              '\s*,\s*'
-            )::varchar[],
-            ''
-          ) assistants
-        SQL
-      end
-
-      def self.insert_into_operations(table_name, sql)
-        <<-SQL
-          insert into #{table_name}(
-            operation,
-            row_number,
-            name,
-            email,
-            phone_number,
-            assistants,
-            extra
-          ) #{sql}
-        SQL
-      end
-
-      def self.insert_into_temp_csv_table(table_name, values)
-        <<-SQL
-          insert into #{table_name}(
-            name,
-            email,
-            phone_number,
-            assistants,
-            extra
-          ) values #{values.join(", ")}
-        SQL
-      end
-    end
-  end
-end