dwh 0.1.1 → 0.3.0

data/lib/dwh/settings/mysql.yml

@@ -24,8 +24,8 @@ extract_minute: 'MINUTE(@exp)'
 extract_year_month: 'CAST(CONCAT(YEAR(@exp), LPAD(MONTH(@exp), 2, "0")) AS UNSIGNED)'
 default_week_start_day: "sunday"
 week_start_day: "monday"
-sunday_week_start_day: "DATE(DATE_SUB(@exp, INTERVAL dayofweek(@exp)-1 DAY ))"
-monday_week_start_day: "DATE(DATE_SUB(@exp, INTERVAL dayofweek(@exp)-2 DAY ))"
+sunday_week_start_day: "DATE(DATE_SUB(@exp, INTERVAL DAYOFWEEK(@exp)-1 DAY ))"
+monday_week_start_day: "DATE(DATE_SUB(@exp, INTERVAL DAYOFWEEK(@exp)-2 DAY ))"
 cast: "CAST(@exp AS @type)"
 
 # string functions

data/lib/dwh/settings/postgres.yml

@@ -1,6 +1,6 @@
 
 date_add: "(@exp + '@val @unit'::interval)"
-date_diff: "age(@start_exp, @end_exp)"
+date_diff: "AGE(@start_exp, @end_exp)"
 date_format_sql: "TO_CHAR(@exp, '@format')"
 date_literal: "'@val'::DATE"
 date_time_literal: "'@val'::TIMESTAMP"
@@ -12,16 +12,16 @@ abbreviated_month_name_format: "Mon"
 sunday_week_start_day: "( DATE_TRUNC('WEEK', @exp + INTERVAL '1 DAY') - INTERVAL '1 DAY' )"
 monday_week_start_day: "( DATE_TRUNC('WEEK', @exp - INTERVAL '1 DAY') + INTERVAL '1 DAY' )"
 
-extract_year: 'extract(year from @exp)'
-extract_month: 'extract(month from @exp)'
-extract_quarter: 'extract(quarter from @exp)'
-extract_day_of_year: 'extract(DOY from @exp)'
-extract_day_of_month: 'extract(DAY from @exp)'
-extract_day_of_week: 'extract(DOW from @exp)'
-extract_week_of_year: 'extract(WEEK from @exp)'
-extract_hour: 'extract(HOUR from @exp)'
-extract_minute: 'extract(MINUTE from @exp)'
-extract_year_month: "cast((extract(year from @exp)::varchar || TO_CHAR(@exp, 'MM')) as integer)"
+extract_year: 'EXTRACT(year from @exp)'
+extract_month: 'EXTRACT(month from @exp)'
+extract_quarter: 'EXTRACT(quarter from @exp)'
+extract_day_of_year: 'EXTRACT(DOY from @exp)'
+extract_day_of_month: 'EXTRACT(DAY from @exp)'
+extract_day_of_week: 'EXTRACT(DOW from @exp)'
+extract_week_of_year: 'EXTRACT(WEEK from @exp)'
+extract_hour: 'EXTRACT(HOUR from @exp)'
+extract_minute: 'EXTRACT(MINUTE from @exp)'
+extract_year_month: "CAST((EXTRACT(YEAR FROM @exp)::varchar || TO_CHAR(@exp, 'MM')) as INTEGER)"
 
 cast: "@exp::@type"
 

data/lib/dwh/settings/redshift.yml

@@ -1,28 +1,19 @@
-
-# quotes and string lit
-quote: "\"@exp\""
-string_literal: "'@exp'"
-
 # Date Literal Formats
-date_format: "%Y-%m-%d"
-date_time_format: "%Y-%m-%d %H:%M:%S"
-date_time_tz_format: "%Y-%m-%d %H:%M:%S %Z"
-date_type: "string" # alternative is int, integer, dateint
 day_name_format: "Day"
 abbreviated_day_name_format: "Dy"
 month_name_format: "Month" 
 abbreviated_month_name_format: "Mon"
 
 # Date functions patterns
-current_date: "current_date"
-current_time: "current_time"
-current_timestamp: "current_timestamp"
-truncate_date: "date_trunc('@unit', @exp)"
-date_add: "dateadd(@unit, @val, @exp)"
-date_diff: "datediff(@unit, @start_exp, @end_exp)"
+current_date: "CURRENT_DATE"
+current_time: "CURRENT_TIME"
+current_timestamp: "CURRENT_TIMESTAMP"
+truncate_date: "DATE_TRUNC('@unit', @exp)"
+date_add: "DATEADD(@unit, @val, @exp)"
+date_diff: "DATEDIFF(@unit, @start_exp, @end_exp)"
 date_format_sql: "TO_CHAR(@exp, '@format')"
-date_literal: "'@val'"
-date_time_literal: "TIMESTAMP '@val'"
+date_literal: "'@val'::DATE"
+date_time_literal: "'@val'::TIMESTAMP"
 extract_year: 'EXTRACT(YEAR FROM @exp)'
 extract_month: 'EXTRACT(MONTH FROM @exp)'
 extract_quarter: 'EXTRACT(QUARTER FROM @exp)'
@@ -33,15 +24,15 @@ extract_week_of_year: 'EXTRACT(WEEK FROM @exp)'
 extract_hour: 'EXTRACT(HOUR FROM @exp)'
 extract_minute: 'EXTRACT(MINUTE FROM @exp)'
 extract_year_month: "TO_CHAR(@exp, 'YYYYMM')::INTEGER"
-default_week_start_day: "sunday" # Redshift uses Sunday as default
-week_start_day: "sunday"
-sunday_week_start_day: "DATEADD(day, -1, DATE_TRUNC(WEEK, DATEADD(DAY, 1, @exp)))"
-monday_week_start_day: "DATEADD(day, 1, DATE_TRUNC(WEEK, DATEADD(day, -1, @exp)))"
+default_week_start_day: "monday" # Redshift uses Sunday as default
+week_start_day: "monday"
+sunday_week_start_day: "DATEADD(day, -1, DATE_TRUNC('WEEK', DATEADD(DAY,  1, @exp)))"
+monday_week_start_day: "DATEADD(day,  1, DATE_TRUNC('WEEK', DATEADD(day, -1, @exp)))"
 
 # string functions
-trim: "trim(@exp)"
-lower_case: "lower(@exp)"
-upper_case: "upper(@exp)"
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
 
 # null handling
 if_null: "COALESCE(@exp, @when_null)"

data/lib/dwh/settings/snowflake.yml

@@ -10,29 +10,29 @@ month_name_format: "MMMM"
 abbreviated_month_name_format: "MON"
 
 # Date functions patterns
-current_date: "current_date()"
-current_time: "current_time()"
-current_timestamp: "current_timestamp()"
+current_date: "CURRENT_DATE()"
+current_time: "CURRENT_TIME()"
+current_timestamp: "CURRENT_TIMESTAMP()"
 
-date_add: "dateadd(@unit, @val, @exp)"
-date_diff: "datediff(@unit, @start_exp, @end_exp)"
+date_add: "DATEADD(@unit, @val, @exp)"
+date_diff: "DATEDIFF(@unit, @start_exp, @end_exp)"
 date_format_sql: "TO_VARCHAR(@exp, '@format')"
 date_literal: "'@val'::DATE"
 date_time_literal: "'@val'::TIMESTAMP"
-extract_year: 'year(@exp)'
-extract_month: 'month(@exp)'
-extract_quarter: 'quarter(@exp)'
+extract_year: 'YEAR(@exp)'
+extract_month: 'MONTH(@exp)'
+extract_quarter: 'QUARTER(@exp)'
 extract_day_of_year: 'DAYOFYEAR(@exp)'
-extract_day_of_month: 'day(@exp)'
+extract_day_of_month: 'DAY(@exp)'
 extract_day_of_week: 'DAYOFWEEK(@exp)'
-extract_week_of_year: 'week(@exp)'
-extract_hour: 'hour(@exp)'
-extract_minute: 'minute(@exp)'
-extract_year_month: 'cast((TO_VARCHAR(@exp, ''YYYY'') || TO_VARCHAR(@exp, ''MM'')) as integer)'
+extract_week_of_year: 'WEEK(@exp)'
+extract_hour: 'HOUR(@exp)'
+extract_minute: 'MINUTE(@exp)'
+extract_year_month: 'CAST((TO_VARCHAR(@exp, ''YYYY'') || TO_VARCHAR(@exp, ''MM'')) as integer)'
 default_week_start_day: "monday"
 week_start_day: "monday"
-sunday_week_start_day: "dateadd(day, -1,date_trunc('week', dateadd(day,1, @exp)))"
-monday_week_start_day: "dateadd(day, 1,date_trunc('week', dateadd(day,-1, @exp)))"
+sunday_week_start_day: "DATEADD(day, -1,DATE_TRUNC('week', DATEADD(day,1, @exp)))"
+monday_week_start_day: "DATEADD(day, 1,DATE_TRUNC('week', DATEADD(day,-1, @exp)))"
 
 # array operations
 array_in_list: "ARRAY_CONTAINS(@exp, ARRAY_CONSTRUCT(@list))"

data/lib/dwh/settings/sqlite.yml

@@ -0,0 +1,42 @@
+
+
+# Date Literal Formats
+day_name_format: "%A"
+abbreviated_day_name_format: "%a"
+month_name_format: "%B"
+abbreviated_month_name_format: "%b"
+
+# Date functions patterns
+# SQLite uses strftime for most date operations
+date_add: "datetime(@exp, '+@val @unit')"
+date_diff: "CAST((julianday(@end_exp) - julianday(@start_exp)) AS INTEGER)"
+date_format_sql: "strftime('@format', @exp)"
+extract_year: "CAST(strftime('%Y', @exp) AS INTEGER)"
+extract_month: "CAST(strftime('%m', @exp) AS INTEGER)"
+extract_quarter: "CAST(((strftime('%m', @exp) - 1) / 3) + 1 AS INTEGER)"
+extract_day_of_year: "CAST(strftime('%j', @exp) AS INTEGER)"
+extract_day_of_month: "CAST(strftime('%d', @exp) AS INTEGER)"
+extract_day_of_week: "CAST(strftime('%w', @exp) AS INTEGER)"
+extract_week_of_year: "CAST(strftime('%W', @exp) AS INTEGER)"
+extract_hour: "CAST(strftime('%H', @exp) AS INTEGER)"
+extract_minute: "CAST(strftime('%M', @exp) AS INTEGER)"
+extract_year_month: "CAST(strftime('%Y%m', @exp) AS INTEGER)"
+default_week_start_day: "monday"
+week_start_day: "monday"
+sunday_week_start_day: "date(@exp, '-' || strftime('%w', @exp) || ' days')"
+monday_week_start_day: "date(@exp, '-' || ((CAST(strftime('%w', @exp) AS INTEGER) + 6) % 7) || ' days')"
+
+cast: "CAST(@exp AS @type)"
+
+# string functions
+trim: "TRIM(@exp)"
+lower_case: "LOWER(@exp)"
+upper_case: "UPPER(@exp)"
+
+# Relevant db capabilities for query gen
+create_temp_table_template: "CREATE TEMP TABLE @table AS \n@sql"
+
+# array operations - SQLite has limited native array support
+# These would need JSON extension or custom implementation
+# array_in_list: "EXISTS(SELECT 1 FROM json_each(@exp) WHERE value IN @list)"
+# array_exclude_list: "NOT EXISTS(SELECT 1 FROM json_each(@exp) WHERE value IN @list)"

data/lib/dwh/settings.rb

@@ -44,7 +44,11 @@ module DWH
         logger.debug "#{adapter_name} Adapter didn't have a settings YAML file. Using only base settings."
       end
 
-      @adapter_settings.symbolize_keys!
+      @adapter_settings.transform_keys! do |key|
+        key.to_sym
+      rescue StandardError
+        key
+      end
     end
 
     # By default settings_file are expected to be in a
@@ -69,7 +73,7 @@ module DWH
     end
 
     def adapter_name
-      name.demodulize.downcase
+      name.split('::').last.downcase
     end
 
     def using_base_settings?

data/lib/dwh/table.rb

@@ -22,16 +22,16 @@ module DWH
 
       @physical_name = parts.last
       @table_stats = table_stats
-      @catalog = catalog
-      @schema = schema
+      @catalog = catalog&.strip
+      @schema = schema&.strip
 
-      @catalog = parts.first if @catalog.nil? && parts.length > 2
+      @catalog = parts.first.strip if @catalog.nil? && parts.length > 2
 
       if @schema.nil?
         if parts.length == 2
-          @schema = parts.first
+          @schema = parts.first.strip
         elsif parts.length > 2
-          @schema = parts[1]
+          @schema = parts[1].strip
         end
       end
 
@@ -50,12 +50,20 @@ module DWH
       [catalog, schema].compact.join('.')
     end
 
+    def catalog?
+      catalog && !catalog.empty?
+    end
+
+    def schema?
+      schema && !schema.empty?
+    end
+
     def catalog_and_schema?
-      catalog && schema
+      catalog? && schema?
     end
 
     def catalog_or_schema?
-      catalog || schema
+      catalog? || schema?
     end
 
     def stats
@@ -82,13 +90,13 @@ module DWH
 
     def self.from_hash_or_json(physical_name, metadata)
       metadata = JSON.parse(metadata) if metadata.is_a?(String)
-      metadata.symbolize_keys!
+      metadata.transform_keys!(&:to_sym)
 
-      stats = TableStats.new(**metadata[:stats].symbolize_keys) if metadata.key?(:stats)
+      stats = TableStats.new(**metadata[:stats]&.transform_keys!(&:to_sym)) if metadata.key?(:stats)
       table = new(physical_name, table_stats: stats)
 
       metadata[:columns]&.each do |col|
-        col.symbolize_keys!
+        col.transform_keys!(&:to_sym)
         table << Column.new(
           name: col[:name],
           data_type: col[:data_type],

data/lib/dwh/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DWH
-  VERSION = '0.1.1'
+  VERSION = '0.3.0'
 end

data/lib/dwh.rb

@@ -1,8 +1,4 @@
 require 'faraday'
-require 'active_support/core_ext/string/inflections'
-require 'active_support/core_ext/hash/keys'
-require 'active_support/core_ext/object/blank'
-require 'active_support/duration'
 
 require_relative 'dwh/version'
 require_relative 'dwh/errors'
@@ -19,7 +15,10 @@ require_relative 'dwh/adapters/snowflake'
 require_relative 'dwh/adapters/my_sql'
 require_relative 'dwh/adapters/sql_server'
 require_relative 'dwh/adapters/duck_db'
+require_relative 'dwh/adapters/sqlite'
 require_relative 'dwh/adapters/athena'
+require_relative 'dwh/adapters/redshift'
+require_relative 'dwh/adapters/databricks'
 
 # DWH encapsulates the full functionality of this gem.
 #
@@ -48,7 +47,10 @@ module DWH
   register(:mysql, Adapters::MySql)
   register(:sqlserver, Adapters::SqlServer)
   register(:duckdb, Adapters::DuckDb)
+  register(:sqlite, Adapters::Sqlite)
   register(:athena, Adapters::Athena)
+  register(:redshift, Adapters::Redshift)
+  register(:databricks, Adapters::Databricks)
 
   # start_reaper
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dwh
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Ajo Abraham
@@ -9,20 +9,6 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 8.0.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 8.0.2
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement
@@ -168,13 +154,16 @@ files:
 - lib/dwh.rb
 - lib/dwh/adapters.rb
 - lib/dwh/adapters/athena.rb
+- lib/dwh/adapters/databricks.rb
 - lib/dwh/adapters/druid.rb
 - lib/dwh/adapters/duck_db.rb
 - lib/dwh/adapters/my_sql.rb
 - lib/dwh/adapters/open_authorizable.rb
 - lib/dwh/adapters/postgres.rb
+- lib/dwh/adapters/redshift.rb
 - lib/dwh/adapters/snowflake.rb
 - lib/dwh/adapters/sql_server.rb
+- lib/dwh/adapters/sqlite.rb
 - lib/dwh/adapters/trino.rb
 - lib/dwh/behaviors.rb
 - lib/dwh/capabilities.rb
@@ -197,6 +186,7 @@ files:
 - lib/dwh/settings/postgres.yml
 - lib/dwh/settings/redshift.yml
 - lib/dwh/settings/snowflake.yml
+- lib/dwh/settings/sqlite.yml
 - lib/dwh/settings/sqlserver.yml
 - lib/dwh/settings/trino.yml
 - lib/dwh/streaming_stats.rb
@@ -225,7 +215,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.1
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Data warehouse adapters for interacting with popular data warehouses.
 test_files: []