data_drain 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/lib/data_drain/engine.rb +5 -2
- data/lib/data_drain/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 05fca5dc293bebcb71247b8659d52933d8a5baca6b73e38ba9d9b21236c02643
|
|
4
|
+
data.tar.gz: c7d89c3fb45dc4deb3b871bd403a2e1b39172977893882df2658deb8eafcb22f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fce9cc7ecc7008136c456ed3442bdd4b6c8171e0b28c37a1ba6bc897866b3e4469dfd7af8285a9b1022c70f00cf8e14a0e16a8fd280038f2a0b604418471be4b
|
|
7
|
+
data.tar.gz: 766e0ea20985a90e39fd84727318b6b1a753bad695af44f78bbf94da088f3aef06c1b428811af27fdb46466c9c694a70974ef113f9ee9bdb90b3a616fe353f9c
|
data/CHANGELOG.md
CHANGED
data/lib/data_drain/engine.rb
CHANGED
|
@@ -81,6 +81,7 @@ module DataDrain
|
|
|
81
81
|
def setup_duckdb
|
|
82
82
|
@duckdb.query("INSTALL postgres; LOAD postgres;")
|
|
83
83
|
@duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
|
|
84
|
+
@duckdb.query("ATTACH '#{@config.duckdb_connection_string}' AS pg_source (TYPE POSTGRES, READ_ONLY)")
|
|
84
85
|
|
|
85
86
|
# 💡 Magia del Adapter: Él sabe si cargar httpfs y setear credenciales o no hacer nada
|
|
86
87
|
@adapter.setup_duckdb(@duckdb)
|
|
@@ -90,7 +91,8 @@ module DataDrain
|
|
|
90
91
|
# @return [Integer]
|
|
91
92
|
def get_postgres_count
|
|
92
93
|
pg_sql = "SELECT COUNT(*) AS row_count FROM public.#{@table_name} WHERE #{base_where_sql}"
|
|
93
|
-
|
|
94
|
+
pg_sql = pg_sql.gsub("'", "''")
|
|
95
|
+
query = "SELECT row_count FROM postgres_query('pg_source', '#{pg_sql}')"
|
|
94
96
|
@duckdb.query(query).first.first
|
|
95
97
|
end
|
|
96
98
|
|
|
@@ -103,11 +105,12 @@ module DataDrain
|
|
|
103
105
|
dest_path = @config.storage_mode.to_sym == :s3 ? "s3://#{@bucket}/#{@folder_name}/" : File.join(@bucket, @folder_name, "")
|
|
104
106
|
|
|
105
107
|
pg_sql = "SELECT #{@select_sql} FROM public.#{@table_name} WHERE #{base_where_sql}"
|
|
108
|
+
pg_sql = pg_sql.gsub("'", "''")
|
|
106
109
|
|
|
107
110
|
query = <<~SQL
|
|
108
111
|
COPY (
|
|
109
112
|
SELECT #{@select_sql}
|
|
110
|
-
FROM postgres_query('
|
|
113
|
+
FROM postgres_query('pg_source', '#{pg_sql}')
|
|
111
114
|
) TO '#{dest_path}'
|
|
112
115
|
(
|
|
113
116
|
FORMAT PARQUET,
|
data/lib/data_drain/version.rb
CHANGED