dwh 0.1.0

data/lib/dwh/settings/druid.yml

@@ -0,0 +1,59 @@
+
+date_add: "TIMESTAMPADD(@unit, @val, @exp)"
+date_diff: "TIMESTAMPDIFF(@unit, @start_exp, @end_exp)"
+date_format_sql: "TIME_FORMAT(@exp, '@format')"
+day_name_format: "EEEE"
+abbreviated_day_name_format: "EEE"
+month_name_format: "MMMM"
+abbreviated_month_name_format: "MMM"
+
+date_literal: "'@val'"
+date_time_literal: "TIMESTAMP '@val'"
+extract_year: "TIME_EXTRACT(@exp, 'YEAR')"
+extract_month: "TIME_EXTRACT(@exp, 'MONTH')"
+extract_quarter: "TIME_EXTRACT(@exp, 'QUARTER')"
+extract_day_of_year: "TIME_EXTRACT(@exp, 'DOY')"
+extract_day_of_month: "TIME_EXTRACT(@exp, 'DAY')"
+extract_day_of_week: "TIME_EXTRACT(@exp, 'DOW')"
+extract_week_of_year: "TIME_EXTRACT(@exp, 'WEEK')"
+extract_hour: "TIME_EXTRACT(@exp, 'HOUR')"
+extract_minute: "TIME_EXTRACT(@exp, 'MINUTE')"
+extract_year_month: "cast((TIME_EXTRACT(@exp, 'YEAR') || TIME_EXTRACT(@exp, 'MONTH')) as integer)"
+default_week_start_day: "monday"
+week_start_day: "sunday"
+sunday_week_start_day: "TIME_FLOOR(@exp, 'P7D', TIMESTAMP '1970-01-04 00:00:00')"
+monday_week_start_day: "TIME_FLOOR(@exp, 'P7D', TIMESTAMP '1970-01-05 00:00:00')"
+
+# string functions
+trim: "trim(@exp)"
+lower_case: "lower(@exp)"
+upper_case: "upper(@exp)"
+
+# Relevant db capabilities
+supports_table_join: true
+supports_full_join: true
+supports_cross_join: true
+supports_common_table_expressions: false
+supports_temp_tables: false
+supports_window_functions: true
+extend_ending_date_to_last_hour_of_day: true
+
+# array operations
+array_in_list: "MV_OVERLAP(@exp, ARRAY[@list])" # list is comma separated
+array_exclude_list: "MV_OVERLAP(@exp, ARRAY[@list]) = 0" # list is comma separated
+array_unnest_join: "CROSS JOIN UNNEST(MV_TO_ARRAY(@exp)) @alias"
+
+#joins
+cross_join: "JOIN @relation ON 1=1"
+
+# null handling
+if_null: "NVL(@exp, @when_null)"
+
+# sql output behavior
+temp_table_type: "subquery" # options cte, subquery, temp
+# Determines how measures across fact universes are combined.
+# Default is full join when supported.
+final_pass_measure_join_type: "full" # inner left right etc
+apply_advanced_filtering_on_array_projections: true # druid needs a having clause or un-nesting
+greedy_apply_date_filters: false
+cross_universe_measure_filtering_strategy: "both" # both, final, intermediate

data/lib/dwh/settings/duckdb.yml

@@ -0,0 +1,44 @@
+
+
+# Date Literal Formats
+day_name_format: "%A"
+abbreviated_day_name_format: "%a"
+month_name_format: "%B"
+abbreviated_month_name_format: "%b"
+
+# Date functions patterns
+date_add: "DATE_ADD(@exp, INTERVAL @val @unit)"
+date_diff: "DATE_DIFF('@unit', @start_exp, @end_exp)"
+date_format_sql: "STRFTIME(@exp, '@format')"
+date_literal: "DATE '@val'"
+date_time_literal: "TIMESTAMP '@val'"
+extract_year: "EXTRACT('YEAR' FROM @exp)"
+extract_month: "EXTRACT('MONTH' FROM @exp)"
+extract_quarter: "EXTRACT('QUARTER' FROM @exp)"
+extract_day_of_year: "EXTRACT('DOY' FROM @exp)"
+extract_day_of_month: "EXTRACT('DAY' FROM @exp)"
+extract_day_of_week: "EXTRACT('DAYOFWEEK' FROM @exp)"
+extract_week_of_year: "EXTRACT('WEEK' FROM @exp)"
+extract_hour: 'HOUR(@exp)'
+extract_minute: 'MINUTE(@exp)'
+extract_year_month: "(STRFTIME(@exp, '%Y') || STRFTIME(@exp, '%m'))::INTEGER"
+default_week_start_day: "monday"
+week_start_day: "monday"
+sunday_week_start_day: "(DATE_TRUNC('week', @exp + 1) - 1)"
+monday_week_start_day: "( DATE_TRUNC('week', @exp - 1) + 1 )"
+
+cast: "@exp::@type"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
+
+# Relevant db capabilities for query gen
+create_temp_table_template: "CREATE TEMP TABLE @table AS \n@sql"
+
+# array operations
+array_in_list: "array_length(array_intersect(@exp, @list)) > 0"
+array_exclude_list: "array_length(array_intersect(@exp, @list)) = 0"
+array_unnest_join: ", LATERAL (SELECT UNNEST(@exp)) AS @alias"
+

data/lib/dwh/settings/mysql.yml

@@ -0,0 +1,67 @@
+# quotes and string lit
+quote: "`@exp`"
+string_literal: "'@exp'"
+
+# Date functions patterns
+current_date: "CURDATE()"
+current_time: "CURTIME()"
+current_timestamp: "NOW()"
+truncate_date: "CUSTOM"
+date_add: "DATE_ADD(@exp, INTERVAL @val @unit)"
+date_diff: "DATEDIFF(@end_exp, @start_exp)"
+date_format_sql: "DATE_FORMAT(@exp, '@format')"
+date_literal: "'@val'"
+date_time_literal: "'@val'"
+extract_year: 'YEAR(@exp)'
+extract_month: 'MONTH(@exp)'
+extract_quarter: 'QUARTER(@exp)'
+extract_day_of_year: 'DAYOFYEAR(@exp)'
+extract_day_of_month: 'DAY(@exp)'
+extract_day_of_week: 'DAYOFWEEK(@exp)'
+extract_week_of_year: 'WEEK(@exp)'
+extract_hour: 'HOUR(@exp)'
+extract_minute: 'MINUTE(@exp)'
+extract_year_month: 'CAST(CONCAT(YEAR(@exp), LPAD(MONTH(@exp), 2, "0")) AS UNSIGNED)'
+default_week_start_day: "sunday"
+week_start_day: "monday"
+sunday_week_start_day: "DATE(DATE_SUB(@exp, INTERVAL dayofweek(@exp)-1 DAY ))"
+monday_week_start_day: "DATE(DATE_SUB(@exp, INTERVAL dayofweek(@exp)-2 DAY ))"
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LCASE(@exp)"
+upper_case: "UCASE(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# Relevant db capabilities for query gen
+supports_table_join: true
+supports_full_join: false # MySQL doesn't support FULL OUTER JOIN
+supports_cross_join: true
+supports_sub_queries: true
+supports_common_table_expressions: true # MySQL 8.0+
+supports_temp_tables: true
+create_temp_table_template: "CREATE TEMPORARY TABLE @table AS \n@sql"
+supports_window_functions: true # MySQL 8.0+
+extend_ending_date_to_last_hour_of_day: false
+
+# array operations (MySQL has limited array support, using JSON functions for MySQL 8.0+)
+array_in_list: "JSON_OVERLAPS(@exp, JSON_ARRAY(@list))" # MySQL 8.0.17+
+array_exclude_list: "NOT JSON_OVERLAPS(@exp, JSON_ARRAY(@list))" # MySQL 8.0.17+
+array_unnest_join: "JOIN JSON_TABLE(@exp, '$[*]' COLUMNS (@alias JSON PATH '$')) AS @alias"
+
+cross_join: "CROSS JOIN @relation"
+
+# sql output behavior
+temp_table_type: "cte" # options cte, subquery, temp
+temp_table_prefix: ""
+
+# Determines how measures across fact universes are combined.
+final_pass_measure_join_type: "left" # inner left right etc
+apply_advanced_filtering_on_array_projections: false
+greedy_apply_date_filters: true
+cross_universe_measure_filtering_strategy: "both" # both, final, intermediate

data/lib/dwh/settings/postgres.yml

@@ -0,0 +1,30 @@
+
+date_add: "(@exp + '@val @unit'::interval)"
+date_diff: "age(@start_exp, @end_exp)"
+date_format_sql: "TO_CHAR(@exp, '@format')"
+date_literal: "'@val'::DATE"
+date_time_literal: "'@val'::TIMESTAMP"
+day_name_format: "Day"
+abbreviated_day_name_format: "Dy"
+month_name_format: "Month"
+abbreviated_month_name_format: "Mon"
+# Adjust week start day
+sunday_week_start_day: "( DATE_TRUNC('WEEK', @exp + INTERVAL '1 DAY') - INTERVAL '1 DAY' )"
+monday_week_start_day: "( DATE_TRUNC('WEEK', @exp - INTERVAL '1 DAY') + INTERVAL '1 DAY' )"
+
+extract_year: 'extract(year from @exp)'
+extract_month: 'extract(month from @exp)'
+extract_quarter: 'extract(quarter from @exp)'
+extract_day_of_year: 'extract(DOY from @exp)'
+extract_day_of_month: 'extract(DAY from @exp)'
+extract_day_of_week: 'extract(DOW from @exp)'
+extract_week_of_year: 'extract(WEEK from @exp)'
+extract_hour: 'extract(HOUR from @exp)'
+extract_minute: 'extract(MINUTE from @exp)'
+extract_year_month: "cast((extract(year from @exp)::varchar || TO_CHAR(@exp, 'MM')) as integer)"
+
+cast: "@exp::@type"
+
+# array operations
+array_in_list: "@exp && ARRAY[@list]" # list is comma separated
+array_exclude_list: "NOT (@exp && ARRAY[@list])" # list is comma separated

data/lib/dwh/settings/redshift.yml

@@ -0,0 +1,52 @@
+
+# quotes and string lit
+quote: "\"@exp\""
+string_literal: "'@exp'"
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "Day"
+abbreviated_day_name_format: "Dy"
+month_name_format: "Month" 
+abbreviated_month_name_format: "Mon"
+
+# Date functions patterns
+current_date: "current_date"
+current_time: "current_time"
+current_timestamp: "current_timestamp"
+truncate_date: "date_trunc('@unit', @exp)"
+date_add: "dateadd(@unit, @val, @exp)"
+date_diff: "datediff(@unit, @start_exp, @end_exp)"
+date_format_sql: "TO_CHAR(@exp, '@format')"
+date_literal: "'@val'"
+date_time_literal: "TIMESTAMP '@val'"
+extract_year: 'EXTRACT(YEAR FROM @exp)'
+extract_month: 'EXTRACT(MONTH FROM @exp)'
+extract_quarter: 'EXTRACT(QUARTER FROM @exp)'
+extract_day_of_year: 'EXTRACT(DOY FROM @exp)'
+extract_day_of_month: 'EXTRACT(DAY FROM @exp)'
+extract_day_of_week: 'EXTRACT(DOW FROM @exp)'
+extract_week_of_year: 'EXTRACT(WEEK FROM @exp)'
+extract_hour: 'EXTRACT(HOUR FROM @exp)'
+extract_minute: 'EXTRACT(MINUTE FROM @exp)'
+extract_year_month: "TO_CHAR(@exp, 'YYYYMM')::INTEGER"
+default_week_start_day: "sunday" # Redshift uses Sunday as default
+week_start_day: "sunday"
+sunday_week_start_day: "DATEADD(day, -1, DATE_TRUNC(WEEK, DATEADD(DAY, 1, @exp)))"
+monday_week_start_day: "DATEADD(day, 1, DATE_TRUNC(WEEK, DATEADD(day, -1, @exp)))"
+
+# string functions
+trim: "trim(@exp)"
+lower_case: "lower(@exp)"
+upper_case: "upper(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# array operations
+supports_array_functions: false

data/lib/dwh/settings/snowflake.yml

@@ -0,0 +1,45 @@
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "DY"
+abbreviated_day_name_format: "DY"
+month_name_format: "MMMM"
+abbreviated_month_name_format: "MON"
+
+# Date functions patterns
+current_date: "current_date()"
+current_time: "current_time()"
+current_timestamp: "current_timestamp()"
+
+date_add: "dateadd(@unit, @val, @exp)"
+date_diff: "datediff(@unit, @start_exp, @end_exp)"
+date_format_sql: "TO_VARCHAR(@exp, '@format')"
+date_literal: "'@val'::DATE"
+date_time_literal: "'@val'::TIMESTAMP"
+extract_year: 'year(@exp)'
+extract_month: 'month(@exp)'
+extract_quarter: 'quarter(@exp)'
+extract_day_of_year: 'DAYOFYEAR(@exp)'
+extract_day_of_month: 'day(@exp)'
+extract_day_of_week: 'DAYOFWEEK(@exp)'
+extract_week_of_year: 'week(@exp)'
+extract_hour: 'hour(@exp)'
+extract_minute: 'minute(@exp)'
+extract_year_month: 'cast((TO_VARCHAR(@exp, ''YYYY'') || TO_VARCHAR(@exp, ''MM'')) as integer)'
+default_week_start_day: "monday"
+week_start_day: "monday"
+sunday_week_start_day: "dateadd(day, -1,date_trunc('week', dateadd(day,1, @exp)))"
+monday_week_start_day: "dateadd(day, 1,date_trunc('week', dateadd(day,-1, @exp)))"
+
+# array operations
+array_in_list: "ARRAY_CONTAINS(@exp, ARRAY_CONSTRUCT(@list))"
+array_exclude_list: "NOT ARRAY_CONTAINS(@exp, ARRAY_CONSTRUCT(@list))"
+apply_advanced_filtering_on_array_projections: false # druid needs a having clause or un-nesting
+array_unnest_join: "LATERAL FLATTEN(INPUT => @exp) @alias"
+
+# null handling
+if_null: "NVL(@exp, @when_null)"
+null_if_zero: "NULLIFZERO(@exp)"

data/lib/dwh/settings/sqlserver.yml

@@ -0,0 +1,80 @@
+# quotes and string lit
+quote: "[@exp]"
+string_literal: "'@exp'"
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "dddd"
+abbreviated_day_name_format: "ddd"
+month_name_format: "mmmm"
+abbreviated_month_name_format: "MMM"
+
+# Date functions patterns
+current_date: "CAST(GETDATE()  DATE)"
+current_time: "CAST(GETDATE() AS TIME)"
+current_timestamp: "GETDATE()"
+truncate_date: "DATETRUNC(@unit, @exp)"
+date_add: "DATEADD(@unit, @val, @exp)"
+date_diff: "DATEDIFF(DAY, @start_exp, @end_exp)"
+date_format_sql: "FORMAT(@exp, '@format')"
+date_literal: "'@val'"
+date_time_literal: "'@val'"
+extract_year: 'YEAR(@exp)'
+extract_month: 'MONTH(@exp)'
+extract_quarter: 'DATEPART(QUARTER, @exp)'
+extract_day_of_year: 'DATEPART(DAYOFYEAR, @exp)'
+extract_day_of_month: 'DAY(@exp)'
+extract_day_of_week: 'DATEPART(WEEKDAY, @exp)'
+extract_week_of_year: 'DATEPART(WEEK, @exp)'
+extract_hour: 'DATEPART(HOUR, @exp)'
+extract_minute: 'DATEPART(MINUTE, @exp)'
+extract_year_month: 'CAST(CONCAT(YEAR(@exp), FORMAT(MONTH(@exp), N''00'')) AS INT)'
+default_week_start_day: "sunday"
+week_start_day: "monday"
+sunday_week_start_day: "CAST(DATEADD(DAY, -(DATEPART(WEEKDAY, @exp) - 1), @exp) AS DATE)"
+monday_week_start_day: "CAST(DATEADD(DAY, -(DATEPART(WEEKDAY, @exp) - 2), @exp) AS DATE)"
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# Relevant db capabilities for query gen
+supports_array_functions: false 
+supports_table_join: true
+supports_full_join: true
+supports_cross_join: true
+supports_sub_queries: true
+supports_common_table_expressions: true
+supports_temp_tables: true
+create_temp_table_template: "CREATE TEMPORARY TABLE @table AS \n@sql"
+supports_window_functions: true
+extend_ending_date_to_last_hour_of_day: false # druid needs this for inclusive filtering
+
+# array operations
+array_in_list: "ANY_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_exclude_list: "NONE_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_unnest_join: "CROSS JOIN UNNEST(@exp) @alias"
+
+# joins
+cross_join: "CROSS JOIN @relation"
+
+# sql output behavior
+temp_table_type: "cte" # options cte, subquery, temp
+temp_table_prefix: ""
+# Determines how measures across fact universes are combined.
+# Default is full join when supported.
+final_pass_measure_join_type: "full" # inner left right etc
+apply_advanced_filtering_on_array_projections: false # druid needs a having clause or un-nesting
+greedy_apply_date_filters: true
+cross_universe_measure_filtering_strategy: "both" # both, final, intermediatet
+

data/lib/dwh/settings/trino.yml

@@ -0,0 +1,77 @@
+
+# Date Literal Formats
+date_format: "%Y-%m-%d"
+date_time_format: "%Y-%m-%d %H:%M:%S"
+date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
+date_type: "string" # alternative is int, integer, dateint
+day_name_format: "%W"
+abbreviated_day_name_format: "%a"
+month_name_format: "%M"
+abbreviated_month_name_format: "%b"
+
+# Date functions patterns
+current_date: "CURRENT_DATE"
+current_time: "CURRENT_TIME"
+current_timestamp: "CURRENT_TIMESTAMP"
+truncate_date: "DATE_TRUNC('@unit', @exp)"
+date_add: "DATE_ADD('@unit', @val, @exp)"
+date_diff: "DATE_DIFF('@unit', @start_exp, @end_exp)"
+date_format_sql: "DATE_FORMAT(@exp, '@format')"
+date_literal: "DATE '@val'"
+date_time_literal: "TIMESTAMP '@val'"
+extract_year: 'YEAR(@exp)'
+extract_month: 'MONTH(@exp)'
+extract_quarter: 'QUARTER(@exp)'
+extract_day_of_year: 'DAY_OF_YEAR(@exp)'
+extract_day_of_month: 'DAY(@exp)'
+extract_day_of_week: 'DAY_OF_WEEK(@exp)'
+extract_week_of_year: 'WEEK(@exp)'
+extract_hour: 'HOUR(@exp)'
+extract_minute: 'MINUTE(@exp)'
+extract_year_month: "CAST(DATE_FORMAT(@exp, '%Y%m') AS INTEGER)"
+default_week_start_day: "monday"
+week_start_day: "monday"
+sunday_week_start_day: "DATE_ADD('day', -1,DATE_TRUNC('week', DATE_ADD('day',1, @exp)))"
+monday_week_start_day: "DATE_ADD('day', 1, DATE_TRUNC('week', DATE_ADD('day',-1, @exp)))"
+
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
+
+# null handling
+if_null: "COALESCE(@exp, @when_null)"
+null_if: "NULLIF(@exp, @target)"
+null_if_zero: "NULLIF(@exp, 0)"
+
+# Relevant db capabilities for query gen
+supports_array_functions: true 
+supports_table_join: true
+supports_full_join: true
+supports_cross_join: true
+supports_sub_queries: true
+supports_common_table_expressions: true
+supports_temp_tables: true
+create_temp_table_template: "CREATE TEMPORARY TABLE @table AS \n@sql"
+supports_window_functions: true
+extend_ending_date_to_last_hour_of_day: false # druid needs this for inclusive filtering
+
+# array operations
+array_in_list: "ANY_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_exclude_list: "NONE_MATCH(@exp, x -> x IN (@list)" # list is comma separated
+array_unnest_join: "CROSS JOIN UNNEST(@exp) @alias"
+
+# joins
+cross_join: "CROSS JOIN @relation"
+
+# sql output behavior
+temp_table_type: "cte" # options cte, subquery, temp
+temp_table_prefix: ""
+# Determines how measures across fact universes are combined.
+# Default is full join when supported.
+final_pass_measure_join_type: "full" # inner left right etc
+apply_advanced_filtering_on_array_projections: false # druid needs a having clause or un-nesting
+greedy_apply_date_filters: true
+cross_universe_measure_filtering_strategy: "both" # both, final, intermediate

data/lib/dwh/settings.rb

@@ -0,0 +1,79 @@
+require 'yaml'
+
+module DWH
+  # Functions related to loading and managing the Adapters
+  # settings. These settings should by default have the same name
+  # as the adapter but lower case and no camelcase.  i.e. MySql adapter
+  # has a settings file called mysql.yml.
+  #
+  # When creating a new adapter copy {settings/base.yml} and modify it
+  # to suit your adapters needs.
+  #
+  # An adapters settings are merged into the base settings config. So each adapter
+  # doesn't have to define every property when they are the same.
+  #
+  # By default the file will be looked for in relative location like settings/myadapter.yml.
+  # However, you can specify the locastion with (@see #settings_file_path)
+  module Settings
+    include Logger
+
+    # Gets the current loaded adatper settings. If nil, then
+    # load_settings hasn't been called.
+    attr_reader :adapter_settings
+
+    # This is the default base settings that each adapter can override
+    # with its own yaml files.
+    BASE_SETTINGS_FILE = File.join(__dir__, 'settings', 'base.yml')
+
+    # This will load adapter level settings. These settings can be
+    # overridden at runtime by calling alter_settings after
+    # an adapter is initialized.
+    def load_settings
+      return unless @adapter_settings.nil?
+
+      logger.debug "+++ LOADING SETTINGS: #{name} +++"
+
+      @using_base = true
+      @adapter_settings = YAML.load_file(BASE_SETTINGS_FILE)
+
+      if File.exist?(settings_file)
+        @using_base = false
+        settings_from_file = YAML.load_file(settings_file) || {}
+        @adapter_settings.merge!(settings_from_file)
+      else
+        logger.debug "#{adapter_name} Adapter didn't have a settings YAML file. Using only base settings."
+      end
+
+      @adapter_settings.symbolize_keys!
+    end
+
+    # By default settings_file are expected to be in a
+    # relative directory called settings. If not,
+    # change the settings file with call to settings_file_path FILE_PATH
+    def settings_file
+      @settings_file ||= File.join(__dir__, 'settings', "#{adapter_name}.yml")
+    end
+
+    # Allows the manual configuration of where to
+    # load default database settings from.
+    #
+    # It will reload settings if adapter settings has already
+    # been loaded.
+    # @param [String] file - path or file name string
+    def settings_file_path(file)
+      @settings_file = file
+      return if @adapter_settings.nil?
+
+      @adapter_settings = nil
+      load_settings
+    end
+
+    def adapter_name
+      name.demodulize.downcase
+    end
+
+    def using_base_settings?
+      @using_base
+    end
+  end
+end

data/lib/dwh/streaming_stats.rb

@@ -0,0 +1,69 @@
+module DWH
+  # Basic streaming stats collector.  This is used when runing a query
+  # via execute_streaming method.  As the data comes in it will write
+  # to the stats object. This can be read in another thread to
+  # update a UI.
+  class StreamingStats
+    # Most cases the streaming data will be streamed to a tempfile
+    # rather than memory. In those cases, we will keep a limited amount
+    # of data in memory.  This is for previewing or other quick intropsections.
+    # The default limit is 20,000 rows.
+    attr_reader :in_memory_limit
+
+    def initialize(limit = 20_000)
+      @status = :init
+      @in_memory_limit = limit
+      @mutex = Mutex.new
+      reset
+    end
+
+    # Resets the data, total rows etc back to 0
+    def reset
+      @mutex.synchronize do
+        @total_rows = 0
+        @max_row_size = 0
+        @data = []
+      end
+    end
+
+    # Add a single row to the in memory dataset. Will
+    # automatically stop once the limit is reached.
+    #
+    # @param [Array] row
+    def <<(row)
+      raise ArgumentError, 'Row must be an array' unless row.is_a?(Array)
+
+      @mutex.synchronize do
+        @data << row unless @data.size >= @in_memory_limit
+        @total_rows += 1
+        @max_row_size = [@max_row_size, row.to_s.bytesize].max
+      end
+    end
+
+    def add_row(row)
+      self << row
+    end
+
+    # Returns the streamed result set thus far upto the
+    # specified limit or default 20,000 rows.
+    def data
+      @mutex.synchronize { @data }
+    end
+
+    # The total rows streamed. This is everything written to the IO object
+    # including whats in memory.
+    def total_rows
+      @mutex.synchronize { @total_rows }
+    end
+
+    # Largest row in bytesize. Can estimate eventual file size.
+    def max_row_size
+      @mutex.synchronize { @max_row_size }
+    end
+
+    # Whether the in memory row limit was reached
+    def limit_reached?
+      data.size >= in_memory_limit
+    end
+  end
+end