dwh 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +7 -0
  2. data/.rubocop.yml +36 -0
  3. data/CHANGELOG.md +5 -0
  4. data/LICENSE +21 -0
  5. data/README.md +130 -0
  6. data/Rakefile +42 -0
  7. data/docs/DWH/Adapters/Adapter.html +3053 -0
  8. data/docs/DWH/Adapters/Athena.html +1704 -0
  9. data/docs/DWH/Adapters/Boolean.html +121 -0
  10. data/docs/DWH/Adapters/Druid.html +1626 -0
  11. data/docs/DWH/Adapters/DuckDb.html +2012 -0
  12. data/docs/DWH/Adapters/MySql.html +1704 -0
  13. data/docs/DWH/Adapters/OpenAuthorizable/ClassMethods.html +265 -0
  14. data/docs/DWH/Adapters/OpenAuthorizable.html +1102 -0
  15. data/docs/DWH/Adapters/Postgres.html +2000 -0
  16. data/docs/DWH/Adapters/Snowflake.html +1662 -0
  17. data/docs/DWH/Adapters/SqlServer.html +2084 -0
  18. data/docs/DWH/Adapters/Trino.html +1835 -0
  19. data/docs/DWH/Adapters.html +129 -0
  20. data/docs/DWH/AuthenticationError.html +142 -0
  21. data/docs/DWH/Behaviors.html +767 -0
  22. data/docs/DWH/Capabilities.html +748 -0
  23. data/docs/DWH/Column.html +1115 -0
  24. data/docs/DWH/ConfigError.html +143 -0
  25. data/docs/DWH/ConnectionError.html +143 -0
  26. data/docs/DWH/DWHError.html +138 -0
  27. data/docs/DWH/ExecutionError.html +143 -0
  28. data/docs/DWH/Factory.html +1133 -0
  29. data/docs/DWH/Functions/Arrays.html +505 -0
  30. data/docs/DWH/Functions/Dates.html +1644 -0
  31. data/docs/DWH/Functions/ExtractDatePart.html +804 -0
  32. data/docs/DWH/Functions/Nulls.html +377 -0
  33. data/docs/DWH/Functions.html +846 -0
  34. data/docs/DWH/Logger.html +258 -0
  35. data/docs/DWH/OAuthError.html +138 -0
  36. data/docs/DWH/Settings.html +658 -0
  37. data/docs/DWH/StreamingStats.html +804 -0
  38. data/docs/DWH/Table.html +1260 -0
  39. data/docs/DWH/TableStats.html +583 -0
  40. data/docs/DWH/TokenExpiredError.html +142 -0
  41. data/docs/DWH/UnsupportedCapability.html +135 -0
  42. data/docs/DWH.html +220 -0
  43. data/docs/_index.html +471 -0
  44. data/docs/class_list.html +54 -0
  45. data/docs/css/common.css +1 -0
  46. data/docs/css/full_list.css +58 -0
  47. data/docs/css/style.css +503 -0
  48. data/docs/file.README.html +210 -0
  49. data/docs/file.adapters.html +514 -0
  50. data/docs/file.creating-adapters.html +497 -0
  51. data/docs/file.getting-started.html +288 -0
  52. data/docs/file.usage.html +446 -0
  53. data/docs/file_list.html +79 -0
  54. data/docs/frames.html +22 -0
  55. data/docs/guides/adapters.md +445 -0
  56. data/docs/guides/creating-adapters.md +430 -0
  57. data/docs/guides/getting-started.md +225 -0
  58. data/docs/guides/usage.md +378 -0
  59. data/docs/index.html +210 -0
  60. data/docs/js/app.js +344 -0
  61. data/docs/js/full_list.js +242 -0
  62. data/docs/js/jquery.js +4 -0
  63. data/docs/method_list.html +2038 -0
  64. data/docs/top-level-namespace.html +110 -0
  65. data/lib/dwh/adapters/athena.rb +359 -0
  66. data/lib/dwh/adapters/druid.rb +267 -0
  67. data/lib/dwh/adapters/duck_db.rb +235 -0
  68. data/lib/dwh/adapters/my_sql.rb +235 -0
  69. data/lib/dwh/adapters/open_authorizable.rb +215 -0
  70. data/lib/dwh/adapters/postgres.rb +250 -0
  71. data/lib/dwh/adapters/snowflake.rb +489 -0
  72. data/lib/dwh/adapters/sql_server.rb +257 -0
  73. data/lib/dwh/adapters/trino.rb +213 -0
  74. data/lib/dwh/adapters.rb +363 -0
  75. data/lib/dwh/behaviors.rb +67 -0
  76. data/lib/dwh/capabilities.rb +39 -0
  77. data/lib/dwh/column.rb +79 -0
  78. data/lib/dwh/errors.rb +29 -0
  79. data/lib/dwh/factory.rb +125 -0
  80. data/lib/dwh/functions/arrays.rb +42 -0
  81. data/lib/dwh/functions/dates.rb +162 -0
  82. data/lib/dwh/functions/extract_date_part.rb +70 -0
  83. data/lib/dwh/functions/nulls.rb +31 -0
  84. data/lib/dwh/functions.rb +86 -0
  85. data/lib/dwh/logger.rb +50 -0
  86. data/lib/dwh/settings/athena.yml +77 -0
  87. data/lib/dwh/settings/base.yml +81 -0
  88. data/lib/dwh/settings/databricks.yml +51 -0
  89. data/lib/dwh/settings/druid.yml +59 -0
  90. data/lib/dwh/settings/duckdb.yml +44 -0
  91. data/lib/dwh/settings/mysql.yml +67 -0
  92. data/lib/dwh/settings/postgres.yml +30 -0
  93. data/lib/dwh/settings/redshift.yml +52 -0
  94. data/lib/dwh/settings/snowflake.yml +45 -0
  95. data/lib/dwh/settings/sqlserver.yml +80 -0
  96. data/lib/dwh/settings/trino.yml +77 -0
  97. data/lib/dwh/settings.rb +79 -0
  98. data/lib/dwh/streaming_stats.rb +69 -0
  99. data/lib/dwh/table.rb +105 -0
  100. data/lib/dwh/table_stats.rb +51 -0
  101. data/lib/dwh/version.rb +5 -0
  102. data/lib/dwh.rb +54 -0
  103. data/sig/dwh.rbs +4 -0
  104. metadata +231 -0
data/docs/file.getting-started.html ADDED
@@ -0,0 +1,288 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>
7
+ File: Getting Started
8
+
9
+ &mdash; Documentation by YARD 0.9.37
10
+
11
+ </title>
12
+
13
+ <link rel="stylesheet" href="css/style.css" type="text/css" />
14
+
15
+ <link rel="stylesheet" href="css/common.css" type="text/css" />
16
+
17
+ <script type="text/javascript">
18
+ pathId = "getting-started";
19
+ relpath = '';
20
+ </script>
21
+
22
+
23
+ <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
24
+
25
+ <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
26
+
27
+
28
+ </head>
29
+ <body>
30
+ <div class="nav_wrap">
31
+ <iframe id="nav" src="file_list.html?1"></iframe>
32
+ <div id="resizer"></div>
33
+ </div>
34
+
35
+ <div id="main" tabindex="-1">
36
+ <div id="header">
37
+ <div id="menu">
38
+
39
+ <a href="_index.html">Index</a> &raquo;
40
+ <span class="title">File: Getting Started</span>
41
+
42
+ </div>
43
+
44
+ <div id="search">
45
+
46
+ <a class="full_list_link" id="class_list_link"
47
+ href="class_list.html">
48
+
49
+ <svg width="24" height="24">
50
+ <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
51
+ <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
52
+ <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
53
+ </svg>
54
+ </a>
55
+
56
+ </div>
57
+ <div class="clear"></div>
58
+ </div>
59
+
60
+ <div id="content"><div id='filecontents'><h1 id="getting-started-with-dwh">Getting Started with DWH</h1>
61
+
62
+ <p>DWH is a lightweight library that provides a unified interface to connect, introspect, and query popular databases. This guide will help you get up and running quickly.</p>
63
+
64
+ <h2 id="installation">Installation</h2>
65
+
66
+ <p>Add this line to your application’s Gemfile:</p>
67
+
68
+ <p><code>ruby
69
+ gem 'dwh'
70
+ </code></p>
71
+
72
+ <p>And then execute:</p>
73
+
74
+ <p><code>bash
75
+ bundle install
76
+ </code></p>
77
+
78
+ <p>Or install it yourself as:</p>
79
+
80
+ <p><code>bash
81
+ gem install dwh
82
+ </code></p>
83
+
84
+ <h2 id="basic-usage">Basic Usage</h2>
85
+
86
+ <h3 id="creating-your-first-connection">Creating Your First Connection</h3>
87
+
88
+ <p>```ruby
89
+ require ‘dwh’</p>
90
+
91
+ <h1 id="connect-to-postgresql">Connect to PostgreSQL</h1>
92
+ <p>postgres = DWH.create(:postgres, {
93
+ host: ‘localhost’,
94
+ database: ‘mydb’,
95
+ username: ‘user’,
96
+ password: ‘password’
97
+ })</p>
98
+
99
+ <h1 id="connect-to-duckdb-in-memory">Connect to DuckDB (in-memory)</h1>
100
+ <p>duckdb = DWH.create(:duckdb, {
101
+ database: ‘:memory:’
102
+ })
103
+ ```</p>
104
+
105
+ <h3 id="your-first-query">Your First Query</h3>
106
+
107
+ <p>```ruby
108
+ # Execute a simple query
109
+ results = postgres.execute(“SELECT * FROM users LIMIT 10”)</p>
110
+
111
+ <h1 id="results-are-returned-as-arrays-by-default">Results are returned as arrays by default</h1>
112
+ <p>results.each do |row|
113
+ puts row.inspect
114
+ end
115
+ ```</p>
116
+
117
+ <h3 id="exploring-your-database">Exploring Your Database</h3>
118
+
119
+ <p>```ruby
120
+ # List all tables
121
+ tables = postgres.tables
122
+ puts “Available tables: #tables.map(&amp;:physical_name)”</p>
123
+
124
+ <h1 id="get-detailed-information-about-a-table">Get detailed information about a table</h1>
125
+ <p>table_info = postgres.metadata(‘users’)
126
+ puts “Table: #table_infotable_info.physical_name”
127
+ puts “Schema: #table_infotable_info.schema”
128
+ puts “Columns:”
129
+ table_info.columns.each do |column|
130
+ puts “ #columncolumn.name (#columncolumn.normalized_data_type)”
131
+ end</p>
132
+
133
+ <h1 id="get-table-statistics">Get table statistics</h1>
134
+ <p>stats = postgres.stats(‘users’, date_column: ‘created_at’)
135
+ puts “Row count: #statsstats.row_count”
136
+ puts “Date range: #statsstats.date_start to #statsstats.date_end”
137
+ ```</p>
138
+
139
+ <h3 id="different-output-formats">Different Output Formats</h3>
140
+
141
+ <p>```ruby
142
+ # Get results as arrays (default)
143
+ array_results = postgres.execute(“SELECT id, name FROM users LIMIT 5”)</p>
144
+
145
+ <h1 id="get-results-as-hashesobjects">Get results as hashes/objects</h1>
146
+ <p>hash_results = postgres.execute(“SELECT id, name FROM users LIMIT 5”, format: :object)</p>
147
+
148
+ <h1 id="get-results-as-csv-string">Get results as CSV string</h1>
149
+ <p>csv_results = postgres.execute(“SELECT id, name FROM users LIMIT 5”, format: :csv)</p>
150
+
151
+ <h1 id="stream-large-results-to-a-file">Stream large results to a file</h1>
152
+ <p>postgres.execute_stream(“SELECT * FROM large_table”, File.open(‘output.csv’, ‘w’))
153
+ ```</p>
154
+
155
+ <h3 id="streaming-large-datasets">Streaming Large Datasets</h3>
156
+
157
+ <p>```ruby
158
+ # stream data while tracting stats and previewing data in a separate thread
159
+ stats = DWH::StreamingStats.new(10000) # num of rows to keep in memory for previewing
160
+ exec_thread = Thread.new {
161
+ postgres.execute_stream(“SELECT * FROM large_table”, File.open(‘output.csv’, ‘w’), stats: stats)
162
+ }</p>
163
+
164
+ <p>mon_thread = Thread.new{
165
+ loop do
166
+ break if exec_thread.alive?</p>
167
+
168
+ <pre class="code ruby"><code class="ruby">puts stats.data.last end }
169
+ </code></pre>
170
+
171
+ <p>[exec_thread, mon_thread].each(&amp;:join)</p>
172
+
173
+ <h1 id="stream-with-block-processing">Stream with block processing</h1>
174
+ <p>postgres.stream(“SELECT * FROM large_table”) do |chunk|
175
+ process_chunk(chunk)
176
+ end</p>
177
+
178
+ <p>```</p>
179
+
180
+ <h2 id="advanced-usage">Advanced Usage</h2>
181
+
182
+ <h3 id="connection-pooling">Connection Pooling</h3>
183
+
184
+ <p>```ruby
185
+ # Create a connection pool
186
+ pool = DWH.pool(‘my_postgres_pool’, :postgres, {
187
+ host: ‘localhost’,
188
+ database: ‘mydb’,
189
+ username: ‘user’,
190
+ password: ‘password’
191
+ }, size: 10, timeout: 5)</p>
192
+
193
+ <h1 id="use-the-pool">Use the pool</h1>
194
+ <p>pool.with do |connection|
195
+ results = connection.execute(“SELECT COUNT(*) FROM users”)
196
+ end</p>
197
+
198
+ <h1 id="shutdown-the-pool-when-done">Shutdown the pool when done</h1>
199
+ <p>DWH.shutdown(‘my_postgres_pool’)
200
+ ```</p>
201
+
202
+ <h3 id="using-extra-connection-params">Using Extra Connection Params</h3>
203
+
204
+ <p>DWH uses an existing Ruby gem where possible to connect to each target database. When that is not possible and the db supports a REST endpoint, we will use Faraday.</p>
205
+
206
+ <p>Using <code>extra_connection_params</code> key you can pass in a Hash of options that the target connector supports but DWH doesn’t make first class. The main config options in DWH are based on required and common needs.</p>
207
+
208
+ <h4 id="sending-postgres-connecttimeout-property-supported-by-the-pg-gem">Sending Postgres ‘connect_timeout’ property supported by the PG gem</h4>
209
+
210
+ <p>```ruby
211
+ pg = DWH.create(:postgres, {
212
+ host: ‘localhost’,
213
+ database: ‘mydb’,
214
+ username: ‘user’,
215
+ password: ‘password’,
216
+ extra_connection_params: {
217
+ connect_timeout: 5
218
+ }
219
+ })</p>
220
+
221
+ <p>```</p>
222
+
223
+ <h3 id="database-functions">Database Functions</h3>
224
+
225
+ <p>DWH provides a function translation layer that converts common SQL functions to database-specific syntax:</p>
226
+
227
+ <p>```ruby
228
+ # Date truncation
229
+ postgres.truncate_date(‘week’, ‘created_at’) # =&gt; DATE_TRUNC(‘week’, created_at)
230
+ sqlserver.truncate_date(‘week’, ‘created_at’) # =&gt; DATETRUNC(week, created_at)</p>
231
+
232
+ <h1 id="date-literals">Date literals</h1>
233
+ <p>postgres.date_literal(‘2025-01-01’) # =&gt; ‘2025-01-01’::DATE
234
+ sqlserver.date_literal(‘2025-01-01’) # =&gt; ‘2025-01-01’</p>
235
+
236
+ <h1 id="null-handling">Null handling</h1>
237
+ <p>adapter.coalesce(‘column1’, ‘column2’, “‘default’”) # =&gt; COALESCE(column1, column2, ‘default’)
238
+ adapter.null_if(‘column1’, “‘empty’”) # =&gt; NULLIF(column1, ‘empty’)</p>
239
+
240
+ <h1 id="string-functions">String functions</h1>
241
+ <p>adapter.trim(‘column_name’) # =&gt; TRIM(column_name)
242
+ adapter.upper_case(‘column_name’) # =&gt; UPPER(column_name)
243
+ adapter.lower_case(‘column_name’) # =&gt; LOWER(column_name)
244
+ ```</p>
245
+
246
+ <h2 id="core-api">Core API</h2>
247
+
248
+ <p>Standardized API across adapters:</p>
249
+
250
+ <dl>
251
+ <dt>connection</dt>
252
+ <dd>creates a reusuable connection based on config hash passed in</dd>
253
+ <dt>tables(schema: nil, catalog: nil)</dt>
254
+ <dd>returns a list of tables from the default connection or from the specified schema and catalog</dd>
255
+ <dt>metadata(table_name, schema: nil, catalog: nil)</dt>
256
+ <dd>provides metadata about a table</dd>
257
+ <dt>stats(table_name, date_column: nil)</dt>
258
+ <dd>provides table row count and date range</dd>
259
+ <dt>execute(sql, format: :array, retries: 0)</dt>
260
+ <dd>runs a query and returns in given format</dd>
261
+ <dt>execute_stream(sql, io, stats: nil)</dt>
262
+ <dd>runs a query and streams it as csv into the given io</dd>
263
+ </dl>
264
+
265
+ <h2 id="error-handling">Error Handling</h2>
266
+
267
+ <p><code>ruby
268
+ begin
269
+ results = adapter.execute("SELECT * FROM non_existent_table")
270
+ rescue DWH::ExecutionError =&gt; e
271
+ puts "Query failed: #{e.message}"
272
+ rescue DWH::ConnectionError =&gt; e
273
+ puts "Connection failed: #{e.message}"
274
+ rescue DWH::ConfigError =&gt; e
275
+ puts "Configuration error: #{e.message}"
276
+ end
277
+ </code></p>
278
+ </div></div>
279
+
280
+ <div id="footer">
281
+ Generated on Fri Aug 22 08:31:21 2025 by
282
+ <a href="https://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
283
+ 0.9.37 (ruby-3.4.4).
284
+ </div>
285
+
286
+ </div>
287
+ </body>
288
+ </html>