dwh 0.1.0 → 0.2.0

data/docs/guides/adapters.md

@@ -70,6 +70,71 @@ postgres = DWH.create(:postgres, {
 })
 ```
 
+## Redshift Adapter
+
+The Redshift adapter uses the `pg` gem and provides full-featured RDBMS support.
+
+### Basic Configuration
+
+```ruby
+redshift = DWH.create(:redshift, {
+  host: 'localhost',
+  port: 5432,                    # Default: 5432
+  database: 'mydb',
+  schema: 'public',              # Default: 'public'
+  username: 'user',
+  password: 'password',
+  client_name: 'My Application'  # Default: 'DWH Ruby Gem'
+})
+```
+
+### SSL Configuration
+
+```ruby
+# Basic SSL
+redshift = DWH.create(:redshift, {
+  host: 'localhost',
+  database: 'mydb',
+  username: 'user',
+  password: 'password',
+  ssl: true,
+  extra_connection_params: {
+    sslmode: 'require'  # disable, prefer, require, verify-ca, verify-full
+  }
+})
+
+# Certificate-based SSL
+redshift = DWH.create(:postgres, {
+  host: 'localhost',
+  database: 'mydb',
+  username: 'user',
+  ssl: true,
+  extra_connection_params: {
+    sslmode: 'verify-full',
+    sslrootcert: '/path/to/ca-cert.pem',
+    sslcert: '/path/to/client-cert.pem',
+    sslkey: '/path/to/client-key.pem'
+  }
+})
+```
+
+### Advanced Configuration
+
+```ruby
+redshift = DWH.create(:redshift, {
+  host: 'localhost',
+  database: 'mydb',
+  username: 'user',
+  password: 'password',
+  query_timeout: 3600,  # seconds, default: 3600
+  extra_connection_params: {
+    application_name: 'Data Analysis Tool',
+    connect_timeout: 10,
+    options: '-c maintenance_work_mem=256MB'
+  }
+})
+```
+
 ## Snowflake
 
 Snowflake adapter use the REST apis (https) to connect and query. This adapter also supports Multi-Database
@@ -287,6 +352,99 @@ duckdb = DWH.create(:duckdb, {
 })
 ```
 
+## SQLite Adapter
+
+The SQLite adapter uses the `sqlite3` gem for lightweight embedded database analytics. It's optimized for analytical workloads with WAL mode enabled by default for better concurrent read performance.
+
+### Basic Configuration
+
+```ruby
+# File-based database
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/my/database.sqlite'
+})
+
+# In-memory database
+sqlite = DWH.create(:sqlite, {
+  file: ':memory:'
+})
+```
+
+### Read-Only Mode
+
+```ruby
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/readonly/database.sqlite',
+  readonly: true
+})
+```
+
+### Performance Optimization
+
+The adapter includes default optimizations for analytical workloads:
+- WAL mode enabled by default for concurrent reads
+- 64MB cache size
+- Memory-mapped I/O (128MB)
+- Temp tables stored in memory
+
+```ruby
+# Customize performance settings
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/my/database.sqlite',
+  timeout: 5000,  # busy timeout in milliseconds, default: 5000
+  pragmas: {
+    cache_size: -128000,      # 128MB cache (negative means KB)
+    mmap_size: 268435456,     # 256MB memory-mapped I/O
+    temp_store: 'MEMORY',     # Store temp tables in memory
+    synchronous: 'NORMAL'     # Faster than FULL, safe with WAL
+  }
+})
+```
+
+### Disable WAL Mode
+
+```ruby
+# Disable WAL mode if needed (e.g., for NFS or network filesystems)
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/my/database.sqlite',
+  enable_wal: false
+})
+```
+
+### Advanced Configuration
+
+```ruby
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/analytics.sqlite',
+  readonly: false,
+  enable_wal: true,           # Default: true
+  timeout: 10000,             # 10 second busy timeout
+  pragmas: {
+    journal_mode: 'WAL',      # Explicitly set WAL (done by default)
+    cache_size: -256000,      # 256MB cache
+    page_size: 8192,          # Larger page size for analytics
+    mmap_size: 536870912,     # 512MB memory-mapped I/O
+    temp_store: 'MEMORY',     # Keep temp data in memory
+    synchronous: 'NORMAL',    # Balance between safety and speed
+    locking_mode: 'NORMAL'    # Allow multiple connections
+  }
+})
+```
+
+### Multiple Connections
+
+Unlike DuckDB, SQLite allows multiple independent connections to the same database file:
+
+```ruby
+# Multiple readers/writers to the same file
+reader = DWH.create(:sqlite, { file: '/path/to/data.sqlite', readonly: true })
+writer = DWH.create(:sqlite, { file: '/path/to/data.sqlite' })
+
+# Both can operate concurrently with WAL mode enabled
+data = reader.execute('SELECT * FROM sales')
+writer.execute('INSERT INTO sales VALUES (...)')
+```
+
 ## Trino Adapter
 
 The Trino adapter requires the `trino-client-ruby` gem and works with both Trino and Presto.

data/docs/guides/getting-started.md

@@ -40,9 +40,14 @@ postgres = DWH.create(:postgres, {
   password: 'password'
 })
 
+# Connect to SQLite (lightweight, embedded)
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/analytics.db'
+})
+
 # Connect to DuckDB (in-memory)
 duckdb = DWH.create(:duckdb, {
-  database: ':memory:'
+  file: ':memory:'
 })
 ```
 

data/docs/guides/usage.md

@@ -293,7 +293,7 @@ native = adapter.execute(sql, format: :native)   # Database's native format
 # Use streaming for large result sets
 def export_large_table(adapter, table_name, output_file)
   query = "SELECT * FROM #{table_name}"
-  
+
   File.open(output_file, 'w') do |file|
     adapter.execute_stream(query, file)
   end
@@ -309,6 +309,38 @@ def process_large_dataset(adapter, query)
 end
 ```
 
+### SQLite Performance Tuning
+
+SQLite adapter comes with optimized defaults for analytical workloads, but can be further tuned:
+
+```ruby
+# High-performance SQLite configuration for analytics
+sqlite = DWH.create(:sqlite, {
+  file: '/path/to/large_analytics.db',
+  enable_wal: true,           # WAL mode for concurrent reads (default: true)
+  timeout: 30000,             # 30 second busy timeout for heavy writes
+  pragmas: {
+    cache_size: -512000,      # 512MB cache for large datasets
+    page_size: 8192,          # Larger pages for sequential scans
+    mmap_size: 1073741824,    # 1GB memory-mapped I/O
+    temp_store: 'MEMORY',     # Keep temp tables in RAM
+    synchronous: 'NORMAL',    # Balance safety/speed (safe with WAL)
+    journal_size_limit: 67108864  # 64MB journal limit
+  }
+})
+
+# Read-only analytics queries with maximum performance
+readonly_analytics = DWH.create(:sqlite, {
+  file: '/path/to/data.db',
+  readonly: true,             # Read-only for maximum concurrency
+  pragmas: {
+    cache_size: -256000,      # 256MB cache
+    mmap_size: 2147483648,    # 2GB memory mapping for large files
+    temp_store: 'MEMORY'      # Fast temp operations
+  }
+})
+```
+
 ## Error Handling and Debugging
 
 ### Comprehensive Error Handling

data/docs/index.html

@@ -57,15 +57,14 @@
         <div class="clear"></div>
       </div>
 
-      <div id="content"><div id='filecontents'>
-<p><a href="https://github.com/stratasite/dwh/actions"><img src="https://github.com/stratasite/dwh/workflows/Ruby/badge.svg" alt="Ruby" /></a></p>
+      <div id="content"><div id='filecontents'><p><a href="https://github.com/stratasite/dwh/actions"><img src="https://github.com/stratasite/dwh/workflows/Ruby/badge.svg" alt="Ruby"></a></p>
 
-<h1 id="dwh---data-warehouse-adapter-library">DWH - Data Warehouse Adapter Library</h1>
+<h1 id="dwh-data-warehouse-adapter-library">DWH - Data Warehouse Adapter Library</h1>
 
 <p>A light weight library to connect, introspect, and query popular databases over a unified interface.  This gem is intended for analtyical workloads.  The library also provides database specific translations for common functions like <code>date_trunc</code>, <code>date_add</code> etc.  The function tranlation is not comprehensive. But, it does provides good coverage for date handling, and some array handling as well.</p>
 
 <blockquote>
-  <p>[!NOTE]
+<p>[!NOTE]
 <strong>This is not an ORM</strong> nor will it cast types to ruby unless the underlying client does it out of the box.  The goal here is to create an Architecture where new databases can be onboarded quickly.</p>
 </blockquote>
 
@@ -78,57 +77,55 @@
 <h2 id="features">Features</h2>
 
 <ul>
-  <li><strong>Unified Interface</strong>: Connect to multiple database types using the same API</li>
-  <li><strong>SQL Function Translation</strong>: Automatically translates common SQL functions to database-specific syntax</li>
-  <li><strong>Connection Pooling</strong>: Built-in connection pool management for high-performance applications</li>
-  <li><strong>Rich Metadata</strong>: Extract table schemas, column information, and statistics</li>
+<li><strong>Unified Interface</strong>: Connect to multiple database types using the same API</li>
+<li><strong>SQL Function Translation</strong>: Automatically translates common SQL functions to database-specific syntax</li>
+<li><strong>Connection Pooling</strong>: Built-in connection pool management for high-performance applications</li>
+<li><strong>Rich Metadata</strong>: Extract table schemas, column information, and statistics</li>
 </ul>
 
 <h2 id="supported-databases">Supported Databases</h2>
 
 <ul>
-  <li><strong>Snowflake</strong> - High performance cloud warehouse</li>
-  <li><strong>Trino</strong> (formerly Presto) - Distributed SQL query engine</li>
-  <li><strong>AWS Athena</strong> - AWS big data warehouse</li>
-  <li><strong>Apache Druid</strong> - Real-time analytics database</li>
-  <li><strong>DuckDB</strong> - In-process analytical database</li>
-  <li><strong>PostgreSQL</strong> - Full-featured RDBMS with advanced SQL support</li>
-  <li><strong>MySQL</strong> - Popular open-source database</li>
-  <li><strong>SQL Server</strong> - Microsoft’s enterprise database</li>
+<li><strong>Snowflake</strong> - High performance cloud warehouse</li>
+<li><strong>Trino</strong> (formerly Presto) - Distributed SQL query engine</li>
+<li><strong>AWS Athena</strong> - AWS big data warehouse</li>
+<li><strong>Apache Druid</strong> - Real-time analytics database</li>
+<li><strong>DuckDB</strong> - In-process analytical database</li>
+<li><strong>PostgreSQL</strong> - Full-featured RDBMS with advanced SQL support</li>
+<li><strong>MySQL</strong> - Popular open-source database</li>
+<li><strong>SQL Server</strong> - Microsoft&#39;s enterprise database</li>
 </ul>
 
 <h2 id="integrations-coming-soon">Integrations Coming Soon</h2>
 
 <ul>
-  <li><strong>Redshift</strong> - AWS data warehouse platform</li>
-  <li><strong>ClickHouse</strong> - High performance analytical db</li>
-  <li><strong>Databricks</strong> - Big data compute engine</li>
-  <li><strong>MotherDuck</strong> - Hosted DuckDB service</li>
+<li><strong>Redshift</strong> - AWS data warehouse platform</li>
+<li><strong>ClickHouse</strong> - High performance analytical db</li>
+<li><strong>Databricks</strong> - Big data compute engine</li>
+<li><strong>MotherDuck</strong> - Hosted DuckDB service</li>
 </ul>
 
 <h2 id="quick-start">Quick Start</h2>
 
 <p>Install it yourself as:</p>
 
-<p><code>bash
-gem install dwh
-</code></p>
+<pre class="code bash"><code class="bash">gem install dwh
+</code></pre>
 
 <h3 id="connect-and-execute-a-basic-query">Connect and Execute a Basic Query</h3>
 
-<p>```ruby
-require ‘dwh’</p>
+<pre class="code ruby"><code class="ruby"><span class='id identifier rubyid_require'>require</span> <span class='tstring'><span class='tstring_beg'>&#39;</span><span class='tstring_content'>dwh</span><span class='tstring_end'>&#39;</span></span>
 
-<h1 id="connect-to-druid">Connect to Druid</h1>
-<p>druid = DWH.create(:druid, {
-  host: ‘localhost’,
-  port: 8080,
-  protocol: ‘http’
-})</p>
+<span class='comment'># Connect to Druid
+</span><span class='id identifier rubyid_druid'>druid</span> <span class='op'>=</span> <span class='const'><span class='object_link'><a href="DWH.html" title="DWH (module)">DWH</a></span></span><span class='period'>.</span><span class='id identifier rubyid_create'><span class='object_link'><a href="DWH/Factory.html#create-instance_method" title="DWH::Factory#create (method)">create</a></span></span><span class='lparen'>(</span><span class='symbol'>:druid</span><span class='comma'>,</span> <span class='lbrace'>{</span>
+  <span class='label'>host:</span> <span class='tstring'><span class='tstring_beg'>&#39;</span><span class='tstring_content'>localhost</span><span class='tstring_end'>&#39;</span></span><span class='comma'>,</span>
+  <span class='label'>port:</span> <span class='int'>8080</span><span class='comma'>,</span>
+  <span class='label'>protocol:</span> <span class='tstring'><span class='tstring_beg'>&#39;</span><span class='tstring_content'>http</span><span class='tstring_end'>&#39;</span></span>
+<span class='rbrace'>}</span><span class='rparen'>)</span>
 
-<h1 id="basic-query-execution">basic query execution</h1>
-<p>results = druid.execute(“SELECT * FROM web_sales”, format: :csv)
-```</p>
+<span class='comment'># basic query execution
+</span><span class='id identifier rubyid_results'>results</span> <span class='op'>=</span> <span class='id identifier rubyid_druid'>druid</span><span class='period'>.</span><span class='id identifier rubyid_execute'>execute</span><span class='lparen'>(</span><span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>SELECT * FROM web_sales</span><span class='tstring_end'>&quot;</span></span><span class='comma'>,</span> <span class='label'>format:</span> <span class='symbol'>:csv</span><span class='rparen'>)</span>
+</code></pre>
 
 <h2 id="core-api">Core API</h2>
 
@@ -152,10 +149,11 @@ require ‘dwh’</p>
 <h2 id="tutorials-and-guides">Tutorials and Guides</h2>
 
 <ul>
-  <li><a href="/docs/guides/getting-started.md">Getting Started</a></li>
-  <li><a href="/docs/guides/adapters.md">Adapter Configuration</a></li>
-  <li><a href="/docs/guides/creating-adapters.md">Creating an Adapter</a></li>
-  <li><a href="https://rubydoc.info/github/stratasite/dwh.git">API</a></li>
+<li><a href="https://strata.site/dwh/file.getting-started.html">Getting Started</a></li>
+<li><a href="https://strata.site/dwh/file.adapters.html">Adapter Configuration</a></li>
+<li><a href="https://strata.site/dwh/file.creating-adapters.html">Creating an Adapter</a></li>
+<li><a href="https://strata.site/dwh/file.usage.html">Advanced Usage</a></li>
+<li><a href="https://strata.site/dwh/DWH.html">API</a></li>
 </ul>
 
 <h2 id="testing">Testing</h2>
@@ -164,21 +162,18 @@ require ‘dwh’</p>
 
 <p>Run Unit Tests:</p>
 
-<p><code>bash
-bundle exec rake test:unit
-</code></p>
+<pre class="code bash"><code class="bash">bundle exec rake test:unit
+</code></pre>
 
 <p>Run tests on RDBMS dbs:</p>
 
-<p><code>bash
-bundle exec rake test:system:rdbms 
-</code></p>
+<pre class="code bash"><code class="bash">bundle exec rake test:system:rdbms 
+</code></pre>
 
 <p>Run tests on  druid:</p>
 
-<p><code>bash
-bundle exec rake test:system:druid 
-</code></p>
+<pre class="code bash"><code class="bash">bundle exec rake test:system:druid 
+</code></pre>
 
 <h2 id="development">Development</h2>
 
@@ -200,7 +195,7 @@ bundle exec rake test:system:druid
 </div></div>
 
       <div id="footer">
-  Generated on Fri Aug 22 08:31:21 2025 by
+  Generated on Mon Aug 25 10:59:27 2025 by
   <a href="https://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
   0.9.37 (ruby-3.4.4).
 </div>

data/docs/top-level-namespace.html

@@ -100,7 +100,7 @@
 </div>
 
       <div id="footer">
-  Generated on Fri Aug 22 08:31:21 2025 by
+  Generated on Mon Aug 25 10:59:27 2025 by
   <a href="https://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
   0.9.37 (ruby-3.4.4).
 </div>

data/lib/dwh/adapters/duck_db.rb

@@ -150,7 +150,7 @@ module DWH
 
       # True if the configuration was setup with a schema.
       def schema?
-        config[:schema].present?
+        !config[:schema].nil? && !config[:schema]&.strip&.empty?
       end
 
       # (see Adapter#execute)

data/lib/dwh/adapters/postgres.rb

@@ -27,7 +27,7 @@ module DWH
       config :schema, String, default: 'public', message: 'schema name. defaults to "public"'
       config :username, String, required: true, message: 'connection username'
       config :password, String, required: false, default: nil, message: 'connection password'
-      config :query_timeout, String, required: false, default: 3600, message: 'query execution timeout in seconds'
+      config :query_timeout, Integer, required: false, default: 3600, message: 'query execution timeout in seconds'
       config :ssl, Boolean, required: false, default: false, message: 'use ssl'
       config :client_name, String, required: false, default: 'DWH Ruby Gem', message: 'The name of the connecting app'
 
@@ -45,7 +45,7 @@ module DWH
           password: config[:password],
           application_name: config[:client_name]
         }.merge(extra_connection_params)
-        properties[:options] = "#{properties[:options]} -c statement_timeout=#{config[:query_timeout]}s"
+        properties[:options] = "#{properties[:options]} -c statement_timeout=#{config[:query_timeout] * 1000}"
 
         @connection = PG.connect(properties)
 
@@ -114,7 +114,7 @@ module DWH
         db_table = Table.new table, schema: qualifiers[:schema]
 
         schema_where = ''
-        if db_table.schema.present?
+        if db_table.schema?
           schema_where = "AND table_schema = '#{db_table.schema}'"
         elsif schema?
           schema_where = "AND table_schema in (#{qualified_schema_name})"
@@ -143,7 +143,7 @@ module DWH
 
       # True if the configuration was setup with a schema.
       def schema?
-        config[:schema].present?
+        !config[:schema].nil? && !config[:schema]&.strip&.empty?
       end
 
       # (see Adapter#execute)

data/lib/dwh/adapters/redshift.rb

@@ -0,0 +1,48 @@
+module DWH
+  module Adapters
+    # Redshift adapter. Please ensure the pg gem is available before using this adapter.
+    # Generally, adapters should be created using {DWH::Factory#create DWH.create}. Where a configuration
+    # is passed in as options hash or argument list.
+    #
+    # @example Basic connection with required only options
+    #   DWH.create(:redshift, {host: 'localhost', database: 'redshift',
+    #     username: 'redshift'})
+    #
+    # @example Connection with cert based SSL connection
+    #   DWH.create(:redshift, {host: 'localhost', database: 'redshift',
+    #     username: 'redshift', ssl: true,
+    #     extra_connection_params: { sslmode: 'require' })
+    #
+    #   valid sslmodes: disable, prefer, require, verify-ca, verify-full
+    #   For modes requiring Certs make sure you add the appropirate params
+    #   to extra_connection_params. (ie sslrootcert, sslcert etc.)
+    #
+    # @example Connection sending custom application name
+    #   DWH.create(:redshift, {host: 'localhost', database: 'redshift',
+    #     username: 'redshift', application_name: "Strata CLI" })
+    class Redshift < Postgres
+      config :host, String, required: true, message: 'server host ip address or domain name'
+      config :port, Integer, required: false, default: 5439, message: 'port to connect to'
+      config :database, String, required: true, message: 'name of database to connect to'
+      config :schema, String, default: 'public', message: 'schema name. defaults to "public"'
+      config :username, String, required: true, message: 'connection username'
+      config :password, String, required: false, default: nil, message: 'connection password'
+      config :query_timeout, Integer, required: false, default: 3600, message: 'query execution timeout in seconds'
+      config :client_name, String, required: false, default: 'DWH Ruby Gem', message: 'The name of the connecting app'
+      config :ssl, Boolean, required: false, default: false, message: 'use ssl'
+
+      # Need to override default add method
+      # since redshift doesn't support quarter as an
+      # interval.
+      # @param unit [String] Should be one of day, month, quarter etc
+      # @param val [String, Integer] The number of days to add
+      # @param exp [String] The sql expresssion to modify
+      def date_add(unit, val, exp)
+        gsk(:date_add)
+          .gsub(/@unit/i, unit)
+          .gsub(/@val/i, val.to_s)
+          .gsub(/@exp/i, exp)
+      end
+    end
+  end
+end

data/lib/dwh/adapters/sql_server.rb

@@ -142,7 +142,7 @@ module DWH
         change_current_database(db_table.catalog)
 
         schema_where = ''
-        schema_where = "AND table_schema = '#{db_table.schema}'" if db_table.schema.present?
+        schema_where = "AND table_schema = '#{db_table.schema}'" if db_table.schema?
 
         sql = <<-SQL
                     SELECT column_name, data_type, character_maximum_length, numeric_precision,numeric_scale