dwh 0.1.0

data/lib/dwh/functions/arrays.rb

@@ -0,0 +1,42 @@
+module DWH
+  module Functions
+    # Operations on array columns. Mostly used in where clauses
+    module Arrays
+      # Generates sql test to see if any values from the passed
+      # in list is in the array column/exp.
+      #
+      # @param exp [String] - sql expression
+      # @param list [String] - comma separated list
+      # @raise UnsupportedCapability if the db doesn't support array functions.
+      def array_in_list(exp, list)
+        raise UnsupportedCapability unless supports_array_functions?
+
+        gsk(:array_in_list).gsub(/@exp/i, exp).gsub(/@list/i, list)
+      end
+
+      # Generates sql test to see if any values from the passed
+      # in list is NOT in the array column/exp.
+      #
+      # @param exp [String] - sql expression
+      # @param list [String] - comma separated list
+      # @raise UnsupportedCapability if the db doesn't support array functions.
+      def array_exclude_list(exp, list)
+        raise UnsupportedCapability unless supports_array_functions?
+
+        gsk(:array_exclude_list).gsub(/@exp/i, exp).gsub(/@list/i, list)
+      end
+
+      # Explode an array. This should be used in a join expression.
+      # @param exp [String] the column/expression that will be Explode
+      # @param table_alias [String] the alias of the exploded array
+      # @raise UnsupportedCapability if the db doesn't support array functions.
+      def array_unnest_join(exp, table_alias)
+        raise UnsupportedCapability unless supports_array_functions?
+
+        gsk(:array_unnest_join)
+          .gsub(/@exp/i, exp)
+          .gsub(/@alias/i, table_alias)
+      end
+    end
+  end
+end

data/lib/dwh/functions/dates.rb

@@ -0,0 +1,162 @@
+module DWH
+  module Functions
+    # Standard date functions except for those dealing with extracting
+    # a date part.
+    module Dates
+      # Ruby format string that is compatible for the target database.
+      def date_format
+        settings[:date_format]
+      end
+
+      # Ruby format string that is compatible timestamp format
+      # for the target db.
+      def date_time_format
+        settings[:date_time_format]
+      end
+
+      # Ruby format string that is compatible timestamp with timezone
+      # format for the target db.
+      def date_time_tz_format
+        settings[:date_time_tz_format]
+      end
+
+      # The native date storage type of the db.
+      def date_data_type
+        settings[:data_type]
+      end
+
+      # Whether the db uses an int format store dates natively
+      def date_int?
+        date_data_type =~ /int/i
+      end
+
+      # The native function to return current date
+      def current_date
+        settings[:current_date]
+      end
+
+      # The native function to return current time
+      def current_time
+        settings[:current_time]
+      end
+
+      # The native function to return current timestamp
+      def current_timestamp
+        settings[:current_timestamp]
+      end
+
+      TIMESTAMPABLE_UNITS = %w[millisecond second minute hour].freeze
+
+      # Given a date expression, truncate it to a given level.
+      # The SQL output of a truncation is still date or timestamp.
+      #
+      # @param unit [String] the unit to truncate to
+      # @param exp [String] the expression to truncate
+      #
+      # @example Truncate date to the week start day
+      #   truncate_date('week', 'my_date_col')
+      #   Postgres ==> DATE_TRUNC('week', 'my_date_col')
+      #   SQL Server ==> DATETRUNC(week, 'my_date_col')
+      #
+      # @note When truncating a literal date rather than an
+      #   expression, the date_literal function should be called on
+      #   it first. e.g. truncate_date('week', date_lit('2025-08-06'))
+      #   For many dbs it won't matter, but some require date literals
+      #   to be specified.
+      def truncate_date(unit, exp)
+        unit = unit.strip.downcase
+        res = if unit == 'week' && adjust_week_start_day?
+                gsk("#{settings[:week_start_day].downcase}_week_start_day")
+                  .gsub(/@exp/i, exp)
+              else
+                gsk(:truncate_date)
+                  .gsub(/@unit/i, unit)
+                  .gsub(/@exp/i, exp)
+
+              end
+
+        # If we are truncating above the timestamp level ie days, years etc
+        # then we can cast the result to date
+        if TIMESTAMPABLE_UNITS.include?(unit)
+          res
+        else
+          cast(res, 'DATE')
+        end
+      end
+
+      # Add some interval of time to a given date expression.
+      # @param unit [String] the units we are adding i.e day, month, week etc
+      # @param val [Integer] the number of said units
+      # @param exp [String] the target expression being operated on
+      def date_add(unit, val, exp)
+        gsk(:date_add)
+          .gsub(/@unit/i, unit)
+          .gsub(/@val/i, val.to_s)
+          .gsub(/@exp/i, exp)
+      end
+
+      # Differnect between two dates in terms of the given unit.
+      # @param unit [String] i.e day, month, hour etc
+      # @param start_exp [String] starting date expression
+      # @param end_exp [String] ending date expression
+      def date_diff(unit, start_exp, end_exp)
+        gsk(:date_diff)
+          .gsub(/@unit/i, unit)
+          .gsub(/@start_exp/i, start_exp)
+          .gsub(/@end_exp/i, end_exp)
+      end
+
+      # Applies the given format to the target date expression
+      def date_format_sql(exp, format)
+        gsk(:date_format_sql)
+          .gsub(/@exp/i, exp)
+          .gsub(/@format/i, format)
+      end
+
+      DATE_CLASSES = [Date, DateTime, Time].freeze
+
+      # Generates a valid date literal string. Most db's
+      # this is just single quoted value while others require
+      # a date declaration.
+      # @param val [String, Date, DateTime, Time]
+      def date_literal(val)
+        val = DATE_CLASSES.include?(val.class) ? val.strftime(date_format) : val
+        gsk(:date_literal).gsub(/@val/i, val)
+      end
+
+      # @param val [String, Date, DateTime, Time]
+      def date_time_literal(val)
+        val = DATE_CLASSES.include?(val.class) ? val.strftime(date_time_format) : val
+        gsk(:date_time_literal).gsub(/@val/i, val)
+      end
+
+      # The current default week start day. This is how
+      # the db is currently setup.  Should be either monday or sunday
+      def default_week_start_day
+        gsk(:default_week_start_day)
+      end
+
+      # The desired week start day. Could be diff from the db setting.
+      def week_start_day
+        gsk(:week_start_day)
+      end
+
+      # Whether we need to adjust the week start day. If its different
+      # from the default, we need to adjust
+      def adjust_week_start_day?
+        week_start_day.strip != default_week_start_day.strip
+      end
+
+      # Apply translation to desired week start day
+      def adjust_week_start_day(exp)
+        gsk("#{settings[:week_start_day].downcase}_week_start_day")
+          .gsub(/@exp/i, exp)
+      end
+
+      # Does the week start on sunday?
+      def week_starts_on_sunday?
+        gsk(:week_start_day) == 'SUNDAY'
+      end
+    end
+  end
+end

data/lib/dwh/functions/extract_date_part.rb

@@ -0,0 +1,70 @@
+module DWH
+  module Functions
+    # All date functions related to extracting part of date
+    # from a date or timestamp.
+    module ExtractDatePart
+      def extract_year(exp)
+        gsk(:extract_year).gsub(/@exp/i, exp)
+      end
+
+      def extract_month(exp)
+        gsk(:extract_month).gsub(/@exp/i, exp)
+      end
+
+      def extract_quarter(exp)
+        gsk(:extract_quarter).gsub(/@exp/i, exp)
+      end
+
+      def extract_day_of_year(exp)
+        gsk(:extract_day_of_year).gsub(/@exp/i, exp)
+      end
+
+      def extract_day_of_month(exp)
+        gsk(:extract_day_of_month).gsub(/@exp/i, exp)
+      end
+
+      def extract_day_of_week(exp)
+        gsk(:extract_day_of_week).gsub(/@exp/i, exp)
+      end
+
+      def extract_week_of_year(exp)
+        gsk(:extract_week_of_year).gsub(/@exp/i, exp)
+      end
+
+      def extract_hour(exp)
+        gsk(:extract_hour).gsub(/@exp/i, exp)
+      end
+
+      def extract_minute(exp)
+        gsk(:extract_minute).gsub(/@exp/i, exp)
+      end
+
+      def extract_year_month(exp)
+        gsk(:extract_year_month).gsub(/@exp/i, exp)
+      end
+
+      def extract_day_name(exp, abbreviate: false)
+        upper_case(
+          if abbreviate
+            date_format_sql(exp,
+                            gsk(:abbreviated_day_name_format))
+          else
+            date_format_sql(exp, gsk(:day_name_format))
+          end
+        )
+      end
+
+      def extract_month_name(exp, abbreviate: false)
+        upper_case(
+          if abbreviate
+            date_format_sql(exp,
+                            gsk(:abbreviated_month_name_format))
+          else
+            date_format_sql(exp,
+                            gsk(:month_name_format))
+          end
+        )
+      end
+    end
+  end
+end

data/lib/dwh/functions/nulls.rb

@@ -0,0 +1,31 @@
+module DWH
+  module Functions
+    # Manages translations to Null handling Functions
+    module Nulls
+      # If the provided expression is null
+      # then return the second expression.
+      # @param exp [String] - target expression
+      # @param exp_if_null [String] - exp if target is null
+      def if_null(exp, exp_if_null)
+        gsk(:if_null)
+          .sub(/@exp/i, exp.to_s)
+          .sub(/@when_null/i, exp_if_null.to_s)
+      end
+
+      # Returns Null if the given expression
+      # equals the target expression.
+      def null_if(exp, target_exp)
+        gsk(:null_if)
+          .sub(/@exp/i, exp.to_s)
+          .sub(/@target/i, target_exp.to_s)
+      end
+
+      # Returns Null if the given expression
+      # evaluates to zero.
+      def null_if_zero(exp)
+        gsk(:null_if_zero)
+          .sub(/@exp/i, exp.to_s)
+      end
+    end
+  end
+end

data/lib/dwh/functions.rb

@@ -0,0 +1,86 @@
+require_relative 'functions/dates'
+require_relative 'functions/extract_date_part'
+require_relative 'functions/nulls'
+require_relative 'functions/arrays'
+
+module DWH
+  # The Functions module adds a translation layer between this library
+  # and native database functions. It relies on the adapters settings
+  # file to map standard function to its native counterpart.
+  #
+  # @example Truncate date to the week start day
+  #   truncate_date('week', 'my_date_col')
+  #   Postgres ==> DATE_TRUNC('week', 'my_date_col')
+  #   SQL Server ==> DATETRUNC(week, 'my_date_col')
+  #
+  # @example Output a date string as a valid date literal
+  #   date_literal('2025-08-06')
+  #   Postgres ==> '2025-08-06'::DATE
+  #   SQL Server ==> '2025-08-06'
+  module Functions
+    include Dates
+    include ExtractDatePart
+    include Nulls
+    include Arrays
+
+    # Casts an expresion/literal to the target datatype.
+    # Datatype should be valid for the target db.
+    #
+    # @param exp [String] sql expression
+    # @param type [String] valid type for target db
+    def cast(exp, type)
+      gsk(:cast).gsub(/@exp/i, exp)
+                .gsub(/@type/i, type)
+    end
+
+    def trim(exp)
+      gsk(:trim).gsub(/@exp/i, exp)
+    end
+
+    def lower_case(exp)
+      gsk(:lower_case).gsub(/@exp/i, exp)
+    end
+
+    def upper_case(exp)
+      gsk(:upper_case).gsub(/@exp/i, exp)
+    end
+
+    # Applies adapter specific quotes around the given
+    # expression. The expression is usually a column name, alias,
+    # or table alias.
+    #
+    # @param exp [String] - column, alias, table name
+    def quote(exp)
+      gsk(:quote).sub(/@exp/i, exp)
+    end
+
+    # Applies adapter specific string literal translation around the given
+    # expression. The expression is usually a string value.
+    #
+    # @param exp [String] some string value
+    def string_lit(exp)
+      gsk(:string_literal).sub(/@exp/i, exp.gsub("'", "''"))
+    end
+
+    # Applies adapter specific cross join expression
+    # This takes the target table. Ie if you are joining
+    # table a with table b.. You will pass in the table b
+    # name expression here.
+    #
+    # @example
+    #   adapter.cross_join("schema.table_b")
+    #
+    # @param relation [String] - table name or table exp
+    def cross_join(relation)
+      gsk(:cross_join).sub(/@relation/i, relation)
+    end
+
+    # Shortcut to get settings value
+    # by key.
+    # @param key [Symbol,String]
+    # @return [String] upcased value
+    def gsk(key)
+      settings[key.to_sym]
+    end
+  end
+end

data/lib/dwh/logger.rb

@@ -0,0 +1,50 @@
+require 'logger'
+
+module DWH
+  # Access a default logger for the gem.
+  module Logger
+    def logger
+      DWH::Logger.logger
+    end
+
+    class << self
+      def logger
+        @logger ||= create_logger
+      end
+
+      private
+
+      def create_logger
+        if rails_logger_available?
+          Rails.logger
+        else
+          standalone_logger
+        end
+      end
+
+      def rails_logger_available?
+        defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      end
+
+      def standalone_logger
+        logger = ::Logger.new($stdout)
+        logger.level = log_level
+        logger.formatter = proc do |severity, datetime, _progname, msg|
+          "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity} DWH: #{msg}\n"
+        end
+        logger
+      end
+
+      def log_level
+        case ENV['DWH_LOG_LEVEL']&.downcase
+        when 'debug' then ::Logger::DEBUG
+        when 'info' then ::Logger::INFO
+        when 'warn' then ::Logger::WARN
+        when 'error' then ::Logger::ERROR
+        when 'fatal' then ::Logger::FATAL
+        else ::Logger::INFO
+        end
+      end
+    end
+  end
+end

data/lib/dwh/settings/athena.yml

@@ -0,0 +1,77 @@
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "%W"
+abbreviated_day_name_format: "%a"
+month_name_format: "%M"
+abbreviated_month_name_format: "%b"
+
+# Date functions patterns
+current_date: "CURRENT_DATE"
+current_time: "CURRENT_TIME"
+current_timestamp: "CURRENT_TIMESTAMP"
+truncate_date: "DATE_TRUNC('@unit', @exp)"
+date_add: "DATE_ADD('@unit', @val, @exp)"
+date_diff: "DATE_DIFF('@unit', @start_exp, @end_exp)"
+date_format_sql: "DATE_FORMAT(@exp, '@format')"
+date_literal: "DATE '@val'"
+date_time_literal: "TIMESTAMP '@val'"
+extract_year: 'YEAR(@exp)'
+extract_month: 'MONTH(@exp)'
+extract_quarter: 'QUARTER(@exp)'
+extract_day_of_year: 'DAY_OF_YEAR(@exp)'
+extract_day_of_month: 'DAY(@exp)'
+extract_day_of_week: 'DAY_OF_WEEK(@exp)'
+extract_week_of_year: 'WEEK(@exp)'
+extract_hour: 'HOUR(@exp)'
+extract_minute: 'MINUTE(@exp)'
+extract_year_month: "CAST(FORMAT('%04d%02d', year(@exp), month(@exp)) AS INTEGER)"
+default_week_start_day: "monday"
+week_start_day: "monday"
+sunday_week_start_day: "DATE_ADD('day', -1,DATE_TRUNC('week', DATE_ADD('day',1, @exp)))"
+monday_week_start_day: "DATE_ADD('day', 1, DATE_TRUNC('week', DATE_ADD('day',-1, @exp)))"
+
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# Relevant db capabilities for query gen
+supports_array_functions: true 
+supports_table_join: true
+supports_full_join: true
+supports_cross_join: true
+supports_sub_queries: true
+supports_common_table_expressions: true
+supports_temp_tables: true
+create_temp_table_template: "CREATE TEMPORARY TABLE @table AS \n@sql"
+supports_window_functions: true
+extend_ending_date_to_last_hour_of_day: false # druid needs this for inclusive filtering
+
+# array operations
+array_in_list: "ANY_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_exclude_list: "NONE_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_unnest_join: "CROSS JOIN UNNEST(@exp) @alias"
+
+# joins
+cross_join: "CROSS JOIN @relation"
+
+# sql output behavior
+temp_table_type: "cte" # options cte, subquery, temp
+temp_table_prefix: ""
+# Determines how measures across fact universes are combined.
+# Default is full join when supported.
+final_pass_measure_join_type: "full" # inner left right etc
+apply_advanced_filtering_on_array_projections: false # druid needs a having clause or un-nesting
+greedy_apply_date_filters: true
+cross_universe_measure_filtering_strategy: "both" # both, final, intermediate

data/lib/dwh/settings/base.yml

@@ -0,0 +1,81 @@
+
+# quotes and string lit
+quote: "\"@exp\""
+string_literal: "'@exp'"
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "%W"
+abbreviated_day_name_format: "%a"
+month_name_format: "%M"
+abbreviated_month_name_format: "%b"
+
+# Date functions patterns
+current_date: "CURRENT_DATE"
+current_time: "CURRENT_TIME"
+current_timestamp: "CURRENT_TIMESTAMP"
+truncate_date: "DATE_TRUNC('@unit', @exp)"
+date_add: "DATE_ADD('@unit', @val, @exp)"
+date_diff: "DATE_DIFF('@unit', @start_exp, @end_exp)"
+date_format_sql: "DATE_FORMAT(@exp, '@format')"
+date_literal: "'@val'"
+date_time_literal: "'@val'"
+extract_year: 'YEAR(@exp)'
+extract_month: 'MONTH(@exp)'
+extract_quarter: 'QUARTER(@exp)'
+extract_day_of_year: 'DAY_OF_YEAR(@exp)'
+extract_day_of_month: 'DAY(@exp)'
+extract_day_of_week: 'DAY_OF_WEEK(@exp)'
+extract_week_of_year: 'WEEK(@exp)'
+extract_hour: 'HOUR(@exp)'
+extract_minute: 'MINUTE(@exp)'
+extract_year_month: 'CAST((year(@exp) || month(@exp)) as integer)'
+default_week_start_day: "monday"
+week_start_day: "monday"
+sunday_week_start_day: "DATE_ADD('day', -1,DATE_TRUNC('week', DATE_ADD('day',1, @exp)))"
+monday_week_start_day: "DATE_ADD('day', 1, DATE_TRUNC('week', DATE_ADD('day',-1, @exp)))"
+
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# Relevant db capabilities for query gen
+supports_array_functions: true 
+supports_table_join: true
+supports_full_join: true
+supports_cross_join: true
+supports_sub_queries: true
+supports_common_table_expressions: true
+supports_temp_tables: true
+create_temp_table_template: "CREATE TEMPORARY TABLE @table AS \n@sql"
+supports_window_functions: true
+extend_ending_date_to_last_hour_of_day: false # druid needs this for inclusive filtering
+
+# array operations
+array_in_list: "ANY_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_exclude_list: "NONE_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_unnest_join: "CROSS JOIN UNNEST(@exp) @alias"
+
+# joins
+cross_join: "CROSS JOIN @relation"
+
+# sql output behavior
+temp_table_type: "cte" # options cte, subquery, temp
+temp_table_prefix: ""
+# Determines how measures across fact universes are combined.
+# Default is full join when supported.
+final_pass_measure_join_type: "full" # inner left right etc
+apply_advanced_filtering_on_array_projections: false # druid needs a having clause or un-nesting
+greedy_apply_date_filters: true
+cross_universe_measure_filtering_strategy: "both" # both, final, intermediate

data/lib/dwh/settings/databricks.yml

@@ -0,0 +1,51 @@
+
+# quotes and string lit
+quote: "\"@exp\""
+string_literal: "'@exp'"
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "EEEE"
+abbreviated_day_name_format: "EEE"
+month_name_format: "MMMM"
+abbreviated_month_name_format: "MMM"
+
+date_add: "date_add(@unit, @val, @exp)"
+date_diff: "date_diff(@unit, @start_exp, @end_exp)"
+date_format_sql: "date_format(@exp, '@format')"
+extract_day_of_year: 'dayofyear(@exp)'
+extract_day_of_week: 'dayofweek(@exp)'
+extract_week_of_year: 'weekofyear(@exp)'
+extract_year_month: 'cast(concat(year(@exp), lpad(month(@exp), 2, "0")) as int)'
+
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "trim(@exp)"
+lower_case: "lower(@exp)"
+upper_case: "upper(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# Relevant db capabilities for query gen
+supports_table_join: true
+supports_full_join: true
+supports_cross_join: true
+supports_sub_queries: true
+supports_common_table_expressions: true
+supports_temp_tables: true
+create_temp_table_template: "CREATE TEMPORARY TABLE @table AS \n@sql"
+supports_window_functions: true
+extend_ending_date_to_last_hour_of_day: false # druid needs this for inclusive filtering
+
+# array operations
+array_in_list: "exists(@exp, x -> x IN (@list))"
+array_exclude_list: "not exists(@exp, x -> x IN (@list))"
+array_unnest_join: "LATERAL VIEW explode(@exp) AS @alias"
+