sp-duh 2.0.6

data/README.md

@@ -0,0 +1,2 @@
+# sp-duh
+Data Uber Helper

data/Rakefile

@@ -0,0 +1,32 @@
+#!/usr/bin/env rake
+#
+# Copyright (c) 2011-2017 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-duh.
+#
+# sp-duh is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-duh is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-duh.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+require "bundler/gem_tasks"
+require 'sp-duh'
+
+load 'lib/tasks/db_utils.rake'
+load 'lib/tasks/i18n.rake'
+load 'lib/tasks/xls2jrxml.rake'
+
+# Allow calling common tasks either from the app (using its environment) or locally from the gem
+# In this case, open the connection to the database
+task :environment => :pg_connect do
+end

data/config/initializers/active_record/connection_adapters_postgre_sql_adapter.rb

@@ -0,0 +1,165 @@
+module ActiveRecord
+  module ConnectionAdapters
+    class PostgreSQLAdapter
+      def active?
+        true
+      end
+
+      def initialize_with_redis_connection(connection, logger, connection_parameters, config)
+        @redis_connection = Rails.application.redis.optimizations.instance_variable_get(:@data)
+        initialize_without_redis_connection(connection, logger, connection_parameters, config)
+      end
+
+      private
+
+        def configure_connection_with_mappings
+          # Build Ruby <-> PostgreSQL type mappings
+          results_map = PG::BasicTypeMapForResults.new(@connection, @redis_connection)
+          queries_map = PG::BasicTypeMapForQueries.new(@connection, @redis_connection)
+
+          # Add a decoder for the NUMERIC PostgreSQL data type
+          results_map.add_coder PG::TextDecoder::Float.new(oid: 1700, name: 'float8')
+          # Fix: Warning: no type cast defined for type "name" with oid 19. Please cast this type explicitly to TEXT to be safe for future changes.
+          results_map.add_coder PG::TextDecoder::String.new(oid: 19, name: 'text')
+          # Fix: Warning: no type cast defined for type "regproc" with oid 24. Please cast this type explicitly to TEXT to be safe for future changes.
+          results_map.add_coder PG::TextDecoder::String.new(oid: 24, name: 'text')
+
+          # Add decoders for the JSON and JSONB data types (using our custom decoder class)
+          results_map.add_coder SP::Duh::Adapters::PG::TextDecoder::Json.new(oid: 114, name: 'json')
+          results_map.add_coder SP::Duh::Adapters::PG::TextDecoder::Json.new(oid: 199, name: '_json')
+          results_map.add_coder SP::Duh::Adapters::PG::TextDecoder::Json.new(oid: 3802, name: 'jsonb')
+          results_map.add_coder SP::Duh::Adapters::PG::TextDecoder::Json.new(oid: 3807, name: '_jsonb')
+
+          # Fix: Remove decoder for Date and Timestamp
+          #<PG::TextDecoder::Date:0x0000000857dcf0 "date"  oid=1082>,
+          results_map.rm_coder(0, 1082)
+
+          #<PG::TextDecoder::TimestampWithoutTimeZone:0x0000000857db38 "timestamp"  oid=1114>,
+          results_map.rm_coder(0, 1114)
+
+          #<PG::TextDecoder::TimestampWithTimeZone:0x0000000857d958 "timestamptz"  oid=1184>,
+          results_map.rm_coder(0, 1184)
+
+          #<PG::TextDecoder::Array:0x0000000868cdf8 "_timestamp"  oid=1115>,
+          results_map.rm_coder(0, 1115)
+
+          #<PG::TextDecoder::Array:0x0000000868cd30 "_date"  oid=1182>,
+          results_map.rm_coder(0, 1182)
+
+          #<PG::TextDecoder::Array:0x0000000868cc40 "_timestamptz"  oid=1185>,
+          results_map.rm_coder(0, 1185)
+
+          # <PG::TextDecoder::Bytea:0x007f96afcc1aa8 "bytea"  oid=17>,
+          results_map.rm_coder(0, 17)
+
+          # Set PostgreSQL type mappings
+          @connection.type_map_for_results = results_map
+          @connection.type_map_for_queries = queries_map
+
+          configure_connection_without_mappings
+        end
+
+        # Override methods to store catalog data in Redis
+        %w(column_definitions primary_key table_exists?).each do |method|
+          method_name, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
+
+          class_eval <<-METHOD, __FILE__, __LINE__ + 1
+            def #{method_name}_with_redis_cache#{punctuation}(table_name)
+              if !!@redis_connection
+                @redis_connection.get("postgresql:#{method_name}:\#{table_name}") || begin
+                  @redis_connection.set("postgresql:#{method_name}:\#{table_name}", d = #{method_name}_without_redis_cache#{punctuation}(table_name)) ; d
+                end
+              else
+                #{method_name}_without_redis_cache#{punctuation}(table_name)
+              end
+            end
+
+            alias_method_chain :#{method}, :redis_cache
+          METHOD
+        end
+
+        def disable_referential_integrity_with_foreign_keys(&block)
+          if Rails.env.test?
+            transaction do
+              begin
+                execute "SET CONSTRAINTS ALL DEFERRED"
+                yield
+              ensure
+                execute "SET CONSTRAINTS ALL IMMEDIATE"
+              end
+            end
+          else
+            disable_referential_integrity_without_foreign_keys &block
+          end
+        end
+
+        alias_method_chain :configure_connection, :mappings
+        alias_method_chain :disable_referential_integrity, :foreign_keys
+        alias_method_chain :initialize, :redis_connection
+    end
+  end
+end
+
+# This module MUST be loaded before the #configure_connection_with_mappings method is called
+module PG
+  class BasicTypeMapForResults
+    def initialize_with_redis_connection(connection, redis_connection = nil)
+      @redis_connection = redis_connection
+      initialize_without_redis_connection connection
+    end
+
+    alias_method_chain :initialize, :redis_connection
+  end
+
+  class BasicTypeMapForQueries
+    def initialize_with_redis_connection(connection, redis_connection = nil)
+      @redis_connection = redis_connection
+      initialize_without_redis_connection connection
+    end
+
+    alias_method_chain :initialize, :redis_connection
+  end
+
+  module BasicTypeRegistry
+
+    private
+
+      def build_coder_maps(connection)
+        if !!@redis_connection
+          @redis_connection.get('postgresql:coder-maps') || begin
+            @redis_connection.set('postgresql:coder-maps', m = get_coder_maps_from_database(connection)) ; m
+          end
+        else
+          get_coder_maps_from_database(connection)
+        end
+      end
+
+      def get_coder_maps_from_database(connection)
+        result = connection.exec(if supports_ranges?(connection)
+          <<-SQL
+            SELECT n.nspname, t.oid, t.typname, t.typelem, t.typdelim, t.typinput, r.rngsubtype
+            FROM pg_type as t
+              LEFT JOIN pg_range as r ON oid = rngtypid
+              LEFT JOIN pg_namespace n ON n.oid = t.typnamespace
+            WHERE n.nspname IN ('pg_catalog', 'public')
+          SQL
+        else
+          <<-SQL
+            SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput
+            FROM pg_type as t
+          SQL
+        end)
+
+        [
+          [0, :encoder, PG::TextEncoder::Array],
+          [0, :decoder, PG::TextDecoder::Array],
+          [1, :encoder, nil],
+          [1, :decoder, nil],
+        ].inject([]) do |h, (format, direction, arraycoder)|
+          h[format] ||= {}
+          h[format][direction] = CoderMap.new result, CODERS_BY_NAME[format][direction], format, arraycoder
+          h
+        end
+      end
+  end
+end

data/config/initializers/active_record/migration_without_transaction.rb

@@ -0,0 +1,4 @@
+module ActiveRecord
+  class MigrationWithoutTransaction < Migration
+  end
+end

data/config/initializers/active_record/migrator.rb

@@ -0,0 +1,34 @@
+module ActiveRecord
+  class Migrator
+    cattr_accessor :current_migration_for_transaction_test
+
+    ActiveRecord::Base.connection.instance_eval do
+      alias :old_supports_ddl_transactions? :supports_ddl_transactions?
+
+      def supports_ddl_transactions?
+        begin
+          require File.expand_path(ActiveRecord::Migrator.current_migration_for_transaction_test.filename)
+
+          if MigrationWithoutTransaction.in?(ActiveRecord::Migrator.current_migration_for_transaction_test.name.constantize.ancestors)
+            false
+          else
+            old_supports_ddl_transactions?
+          end
+        rescue
+          old_supports_ddl_transactions?
+        end
+      end
+    end
+
+    alias_method :old_migrate, :migrate
+
+    def migrate(&block)
+      ActiveRecord::Base.connection.raw_connection.type_map_for_results = PG::TypeMapAllStrings.new
+      ActiveRecord::Base.connection.raw_connection.type_map_for_queries = PG::TypeMapAllStrings.new
+
+      old_migrate do |migration|
+        (block.nil? ? true : block.call) && !(ActiveRecord::Migrator.current_migration_for_transaction_test = migration).nil?
+      end
+    end
+  end
+end

data/config/initializers/rails/generators.rb

@@ -0,0 +1,13 @@
+require 'rails/generators'
+
+module Rails
+  module Generators
+    class <<self
+      alias_method :original_invoke, :invoke
+    end
+
+    def self.invoke(namespace, args=ARGV, config={})
+      original_invoke namespace.tr('-', '_'), args, config
+    end
+  end
+end

data/config/jsonapi/settings.yml

@@ -0,0 +1,14 @@
+version: false
+compound: false
+show-links: false
+show-null: true
+type-restriction: true
+attribute-restriction: false
+empty-is-null: true
+page-size: 0
+request-company-schema: false
+request-sharded-schema: false
+request-accounting-schema: false
+request-accounting-prefix: false
+pg-order-by: "created_at"
+pg-search_path: '"$user", request-sharded-schema, public'

data/config/locales/pt.yml

@@ -0,0 +1,15 @@
+pt:
+  sp-duh:
+    exceptions:
+      jsonapi:
+        service_setup_error: 'O serviço JSONAPI não se encontra disponível porque os recursos não estão devidamente instalados.'
+        service_protocol_error: 'O protocolo definido para acesso ao serviço JSONAPI (%{protocol}) não é suportado. Os protocolos suportados são: %{protocols}.'
+        invalid_resource_configuration_error: 'A configuração de recursos definida no ficheiro %{file} é inválida.'
+        duplicate_resource_error: 'Tentativa de redefinção do recurso %{name}. O recurso já foi definido anteriormente.'
+        save_configuration_error: 'Erro ao gravar a configuração da JSONAPI na base de dados.'
+        invalid_resource_publisher_error: 'O módulo ou classe %{publisher} não é uma fonte de recursos JSONAPI válida.'
+        invalid_jsonapi_key_error: 'A chave %{key} não é válida para configurar a JSONAPI.'
+      sharding:
+        invalid_sharder_type_error: 'Erro de definição do particionamento para a partição %{namespace}: o tipo de partição pretendido, %{sharder_type}, não se encontra instalado.'
+        invalid_sharding_definition_error: 'Erro de definição do particionamento para a partição %{namespace}: não é possível aceder à estrutura de particionamento definida (%{shard_value_field} na estrutura %{shard_table} pelo id %{shard_id_field}).'
+        shard_not_found_error: 'Não existe a partição com o identificador %{shard_id} na estrutura %{shard_table} da partição %{namespace}.'

data/lib/generators/accounting_migration/accounting_migration_generator.rb

@@ -0,0 +1,10 @@
+require 'rails/generators/active_record/migration/migration_generator'
+
+class AccountingMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.expand_path('../templates', __FILE__)
+
+  # def create_migration_file
+  #   set_local_assigns!
+  #   migration_template "migration.rb", "db/migrate/#{file_name}.rb"
+  # end
+end

data/lib/generators/accounting_migration/templates/migration.rb

@@ -0,0 +1,42 @@
+class <%= migration_class_name %> < ActiveRecord::MigrationWithoutTransaction
+  def up
+    schema = 'accounting'
+    ['base_', 'small_', 'micro_', 'independent_', 'nonprofit_'].each do |table_prefix|
+      # Code to run on "accounting" schema
+
+    end
+
+    migrate_user_templates do |schema, table_prefix, ut|
+      # Code to run on user templates' schemas
+
+      # IF CHANGING TABLE we need to keep low cpu usage on DB
+      sleep 0.050
+    end
+
+    migrate_fiscal_years do |schema, table_prefix, fy|
+      # Code to run on fiscal years' schemas
+
+      # IF CHANGING TABLE we need to keep low cpu usage on DB
+      sleep 0.050
+    end
+  end
+
+  def down
+    schema = 'accounting'
+    ['base_', 'small_', 'micro_', 'independent_', 'nonprofit_'].each do |table_prefix|
+      # Code to run on "accounting" schema
+
+    end
+
+    rollback_user_templates do |schema, table_prefix, ut|
+      # Code to run on user templates' schemas
+
+    end
+
+    rollback_fiscal_years do |schema, table_prefix, fy|
+      # Code to run on fiscal years' schemas
+
+    end
+  end
+
+end

data/lib/generators/accounting_payroll_migration/accounting_payroll_migration_generator.rb

@@ -0,0 +1,10 @@
+require 'rails/generators/active_record/migration/migration_generator'
+
+class AccountingPayrollMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.expand_path('../templates', __FILE__)
+
+  # def create_migration_file
+  #   set_local_assigns!
+  #   migration_template "migration.rb", "db/migrate/#{file_name}.rb"
+  # end
+end

data/lib/generators/accounting_payroll_migration/templates/migration.rb

@@ -0,0 +1,73 @@
+class <%= migration_class_name %> < ActiveRecord::MigrationWithoutTransaction
+  def up
+    migrate_fiscal_years do |schema, table_prefix, fy, company_id|
+      result = execute %Q[ SELECT sharded_schema FROM sharding.get_sharded_schema_name(#{company_id}); ]
+      sharded_schema = result.first["sharded_schema"]
+      execute <<-SQL
+        DO $BODY$
+        DECLARE
+        BEGIN
+
+            WITH _needs_update AS (
+              SELECT tcda.id,
+                     accounting.get_account_best_match('#{schema}','#{table_prefix}',COALESCE(tcda.social_organ_account::text,po.social_organ_account,pi.social_organ_account)) AS social_organ_account,
+                     accounting.get_account_best_match('#{schema}','#{table_prefix}',COALESCE(tcda.employee_account::text,po.employee_account,pi.employee_account)) AS employee_account
+                FROM accounting.transaction_suggestions_categories tc
+                JOIN public.payroll_items pi ON pi.type = tc.id
+                LEFT JOIN public.payroll_item_overrides po ON (po.company_id = #{company_id} AND po.payroll_item_id = pi.id)
+                JOIN #{schema}.#{table_prefix}transaction_suggestions_default_accounts tcda ON (tcda.category_id = pi.type AND tcda.item_id = pi.id)
+               WHERE ( pi.company_id = #{company_id} OR pi.company_id IS NULL )
+                 AND (tcda.social_organ_account IS NULL OR tcda.employee_account IS NULL)
+            )
+            UPDATE #{schema}.#{table_prefix}transaction_suggestions_default_accounts
+               SET social_organ_account =  _needs_update.social_organ_account,
+                   employee_account =  _needs_update.employee_account
+              FROM _needs_update
+             WHERE #{schema}.#{table_prefix}transaction_suggestions_default_accounts.id = _needs_update.id
+               AND (_needs_update.social_organ_account IS NOT NULL OR _needs_update.employee_account IS NOT NULL);
+
+            INSERT INTO #{schema}.#{table_prefix}transaction_suggestions_default_accounts
+                   (category_id, item_id, social_organ_account, employee_account, use_employee_account)
+            SELECT pi.type AS category_id,
+                   pi.id AS item_id,
+                   accounting.get_account_best_match('#{schema}','#{table_prefix}',COALESCE(po.social_organ_account,pi.social_organ_account)) AS social_organ_account,
+                   accounting.get_account_best_match('#{schema}','#{table_prefix}',COALESCE(po.employee_account,pi.employee_account)) AS employee_account,
+                   COALESCE(po.use_employee_account,pi.use_employee_account) AS use_employee_account
+              FROM accounting.transaction_suggestions_categories tc
+              JOIN public.payroll_items pi ON pi.type = tc.id
+              LEFT JOIN public.payroll_item_overrides po ON (po.company_id = #{company_id} AND po.payroll_item_id = pi.id)
+              LEFT JOIN #{schema}.#{table_prefix}transaction_suggestions_default_accounts tcda ON (tcda.category_id = pi.type AND tcda.item_id = pi.id)
+             WHERE ( pi.company_id = #{company_id} OR pi.company_id IS NULL )
+               AND tcda IS NULL
+             ORDER BY pi.id;
+
+            INSERT INTO #{schema}.#{table_prefix}transaction_suggestions_default_accounts
+                   (category_id, item_id, social_organ_account, employee_account, use_employee_account)
+            SELECT CASE
+                        WHEN paa.category = 'Taxes'         THEN 'Payroll::Tax'
+                        WHEN paa.category = 'Third parties' THEN 'Payroll::ThirdParty'
+                    END AS category_id,
+                   paa.id AS item_id,
+                   accounting.get_account_best_match('#{schema}','#{table_prefix}',paa.social_organ_account) AS social_organ_account,
+                   accounting.get_account_best_match('#{schema}','#{table_prefix}',paa.employee_account) AS employee_account,
+                   paa.use_employee_account
+              FROM #{sharded_schema}.payroll_accounting_accounts paa
+              LEFT JOIN #{schema}.#{table_prefix}transaction_suggestions_default_accounts tcda ON (tcda.category_id IN ('Payroll::Tax','Payroll::ThirdParty') AND tcda.item_id = paa.id)
+             WHERE paa.company_id = #{company_id}
+               AND paa.category IN ('Taxes', 'Third parties')
+               AND paa.code NOT IN ('BBNK','BCSH')
+               AND tcda IS NULL
+             ORDER BY paa.id;
+        END;
+        $BODY$ LANGUAGE 'plpgsql';
+      SQL
+    end
+  end
+
+  def down
+    puts "Not reverting"
+    rollback_fiscal_years {}
+
+  end
+
+end

data/lib/generators/sharded_migration/sharded_migration_generator.rb

@@ -0,0 +1,10 @@
+require 'rails/generators/active_record/migration/migration_generator'
+
+class ShardedMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.expand_path('../templates', __FILE__)
+
+  # def create_migration_file
+  #   set_local_assigns!
+  #   migration_template "migration.rb", "db/migrate/#{file_name}.rb"
+  # end
+end

data/lib/generators/sharded_migration/templates/migration.rb

@@ -0,0 +1,45 @@
+class <%= migration_class_name %> < ActiveRecord::MigrationWithoutTransaction
+  def up
+    migrate_companies do |schema_name, company_id|
+      case schema_name
+        when 'public'
+          # Code to run on "public" schema
+
+        when 'sharded'
+          # Code to run on "sharded" schema
+          # STUCTURE CHANGES TO TABLES IN THIS SCHEMA MUST ONLY RUN IF ADDING OR REMOVING COLUMNS, NOTHING ELSE!
+
+        else
+          # Code to run on sharded companies' schemas
+
+      end
+      # IF CHANGING TABLE we need to keep low cpu usage on DB
+      sleep 0.100
+    end
+
+    # If migration add structural changes on public objects and want run migration
+    # withou a full deploy, please uncomment the next line
+    # invalidate_postgresql_redis_cache!
+  end
+
+  def down
+    rollback_companies do |schema_name, company_id|
+      case schema_name
+        when 'public'
+          # Code to run on "public" schema
+
+        when 'sharded'
+          # Code to run on "sharded" schema
+          # STUCTURE CHANGES TO TABLES IN THIS SCHEMA MUST ONLY RUN IF ADDING OR REMOVING COLUMNS, NOTHING ELSE!
+
+        else
+          # Code to run on sharded companies' schemas
+
+      end
+    end
+
+    # If migration add structural changes on public objects and want run migration
+    # withou a full deploy, please uncomment the next line
+    # invalidate_postgresql_redis_cache!
+  end
+end