data_drain 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +2 -0
- data/lib/data_drain/configuration.rb +2 -1
- data/lib/data_drain/engine.rb +3 -3
- data/lib/data_drain/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d7c2acf6629c9636f5ea3e56d8fe903197058c072d778226422e9500497284f8
|
|
4
|
+
data.tar.gz: 9099820c61b9601d39bd55bc13297453b8587327b744429a86e69b17eb91320f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8536c37bff85564fa97800876196407ad59aec122ca563bfdc9dd84dc15692c8a3fcc23e7b71a2d22514fefbd9e8ccb6685a8c09066b6549fb504bde56dfb1e2
|
|
7
|
+
data.tar.gz: fe2385046cab1246d2facbd26508a3763fb8bba46c3236889c143f868c070f52d8a2b719877370d12a224c4a05b5d1f1bcf8b35cb831323e23ade923e821da89
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
+
## [0.1.2] - 2026-03-16
|
|
4
|
+
|
|
5
|
+
- Cambiamos postgres_scan por postgres_query
|
|
6
|
+
|
|
7
|
+
## [0.1.1] - 2026-03-16
|
|
8
|
+
|
|
9
|
+
- Se agrega al configure la posibliidad de agregar el limit de ram para duckdb.
|
|
10
|
+
|
|
3
11
|
## [0.1.0] - 2026-03-11
|
|
4
12
|
|
|
5
13
|
- Initial release
|
data/README.md
CHANGED
|
@@ -8,7 +8,7 @@ module DataDrain
|
|
|
8
8
|
attr_accessor :storage_mode, :aws_region,
|
|
9
9
|
:aws_access_key_id, :aws_secret_access_key,
|
|
10
10
|
:db_host, :db_port, :db_user, :db_pass, :db_name,
|
|
11
|
-
:batch_size, :throttle_delay, :logger
|
|
11
|
+
:batch_size, :throttle_delay, :logger, :limit_ram
|
|
12
12
|
|
|
13
13
|
def initialize
|
|
14
14
|
@storage_mode = :local
|
|
@@ -16,6 +16,7 @@ module DataDrain
|
|
|
16
16
|
@db_port = 5432
|
|
17
17
|
@batch_size = 5000
|
|
18
18
|
@throttle_delay = 0.5
|
|
19
|
+
@limit_ram = nil # eg 2GB
|
|
19
20
|
@logger = Logger.new($stdout)
|
|
20
21
|
end
|
|
21
22
|
|
data/lib/data_drain/engine.rb
CHANGED
|
@@ -80,7 +80,7 @@ module DataDrain
|
|
|
80
80
|
# @api private
|
|
81
81
|
def setup_duckdb
|
|
82
82
|
@duckdb.query("INSTALL postgres; LOAD postgres;")
|
|
83
|
-
@duckdb.query("SET max_memory='
|
|
83
|
+
@duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
|
|
84
84
|
|
|
85
85
|
# 💡 Magia del Adapter: Él sabe si cargar httpfs y setear credenciales o no hacer nada
|
|
86
86
|
@adapter.setup_duckdb(@duckdb)
|
|
@@ -91,7 +91,7 @@ module DataDrain
|
|
|
91
91
|
def get_postgres_count
|
|
92
92
|
query = <<~SQL
|
|
93
93
|
SELECT COUNT(*)
|
|
94
|
-
FROM
|
|
94
|
+
FROM postgres_query('#{@config.duckdb_connection_string}', 'public', '#{@table_name}')
|
|
95
95
|
WHERE #{base_where_sql}
|
|
96
96
|
SQL
|
|
97
97
|
@duckdb.query(query).first.first
|
|
@@ -108,7 +108,7 @@ module DataDrain
|
|
|
108
108
|
query = <<~SQL
|
|
109
109
|
COPY (
|
|
110
110
|
SELECT #{@select_sql}
|
|
111
|
-
FROM
|
|
111
|
+
FROM postgres_query('#{@config.duckdb_connection_string}', 'public', '#{@table_name}')
|
|
112
112
|
WHERE #{base_where_sql}
|
|
113
113
|
) TO '#{dest_path}'
|
|
114
114
|
(
|
data/lib/data_drain/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: data_drain
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gabriel
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-03-
|
|
11
|
+
date: 2026-03-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activemodel
|