deltalake-rb 0.1.0 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/Cargo.lock +506 -337
- data/README.md +33 -3
- data/ext/deltalake/Cargo.toml +7 -4
- data/ext/deltalake/src/error.rs +62 -15
- data/ext/deltalake/src/features.rs +67 -0
- data/ext/deltalake/src/lib.rs +1114 -48
- data/ext/deltalake/src/merge.rs +205 -0
- data/lib/deltalake/table.rb +170 -10
- data/lib/deltalake/table_alterer.rb +58 -0
- data/lib/deltalake/table_merger.rb +38 -0
- data/lib/deltalake/table_optimizer.rb +67 -0
- data/lib/deltalake/utils.rb +59 -0
- data/lib/deltalake/version.rb +1 -1
- data/lib/deltalake.rb +50 -12
- metadata +8 -2
data/lib/deltalake.rb
CHANGED
@@ -7,12 +7,17 @@ end
|
|
7
7
|
|
8
8
|
# stdlib
|
9
9
|
require "json"
|
10
|
+
require "time"
|
10
11
|
|
11
12
|
# modules
|
12
13
|
require_relative "deltalake/field"
|
13
14
|
require_relative "deltalake/metadata"
|
14
15
|
require_relative "deltalake/schema"
|
15
16
|
require_relative "deltalake/table"
|
17
|
+
require_relative "deltalake/table_alterer"
|
18
|
+
require_relative "deltalake/table_merger"
|
19
|
+
require_relative "deltalake/table_optimizer"
|
20
|
+
require_relative "deltalake/utils"
|
16
21
|
require_relative "deltalake/version"
|
17
22
|
|
18
23
|
module DeltaLake
|
@@ -22,8 +27,43 @@ module DeltaLake
|
|
22
27
|
class CommitFailedError < Error; end
|
23
28
|
class SchemaMismatchError < Error; end
|
24
29
|
|
25
|
-
class
|
30
|
+
class Todo < Error
|
31
|
+
def message
|
32
|
+
"not implemented yet"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
ProtocolVersions =
|
37
|
+
Struct.new(
|
38
|
+
:min_reader_version,
|
39
|
+
:min_writer_version,
|
40
|
+
:writer_features,
|
41
|
+
:reader_features
|
42
|
+
)
|
43
|
+
|
44
|
+
CommitProperties =
|
45
|
+
Struct.new(
|
46
|
+
:custom_metadata,
|
47
|
+
:max_commit_retries,
|
48
|
+
# TODO
|
49
|
+
# :app_transactions,
|
50
|
+
keyword_init: true
|
51
|
+
)
|
52
|
+
|
53
|
+
PostCommitHookProperties =
|
54
|
+
Struct.new(
|
55
|
+
:create_checkpoint,
|
56
|
+
:cleanup_expired_logs,
|
57
|
+
keyword_init: true
|
58
|
+
)
|
26
59
|
|
60
|
+
class ArrowArrayStream
|
61
|
+
def arrow_c_stream
|
62
|
+
self
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
class << self
|
27
67
|
def write(
|
28
68
|
table_or_uri,
|
29
69
|
data,
|
@@ -35,7 +75,10 @@ module DeltaLake
|
|
35
75
|
schema_mode: nil,
|
36
76
|
storage_options: nil,
|
37
77
|
predicate: nil,
|
38
|
-
target_file_size: nil
|
78
|
+
target_file_size: nil,
|
79
|
+
writer_properties: nil,
|
80
|
+
commit_properties: nil,
|
81
|
+
post_commithook_properties: nil
|
39
82
|
)
|
40
83
|
table, table_uri = try_get_table_and_table_uri(table_or_uri, storage_options)
|
41
84
|
|
@@ -47,7 +90,7 @@ module DeltaLake
|
|
47
90
|
return
|
48
91
|
end
|
49
92
|
|
50
|
-
data = convert_data(data)
|
93
|
+
data = Utils.convert_data(data)
|
51
94
|
|
52
95
|
write_deltalake_rust(
|
53
96
|
table_uri,
|
@@ -61,7 +104,10 @@ module DeltaLake
|
|
61
104
|
name,
|
62
105
|
description,
|
63
106
|
configuration,
|
64
|
-
storage_options
|
107
|
+
storage_options,
|
108
|
+
writer_properties,
|
109
|
+
commit_properties,
|
110
|
+
post_commithook_properties
|
65
111
|
)
|
66
112
|
|
67
113
|
if table
|
@@ -92,13 +138,5 @@ module DeltaLake
|
|
92
138
|
rescue TableNotFoundError
|
93
139
|
nil
|
94
140
|
end
|
95
|
-
|
96
|
-
def convert_data(data)
|
97
|
-
if data.respond_to?(:arrow_c_stream)
|
98
|
-
data.arrow_c_stream
|
99
|
-
else
|
100
|
-
raise TypeError, "Only objects implementing the Arrow C stream interface are valid inputs for source."
|
101
|
-
end
|
102
|
-
end
|
103
141
|
end
|
104
142
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deltalake-rb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rb_sys
|
@@ -39,7 +39,9 @@ files:
|
|
39
39
|
- ext/deltalake/Cargo.toml
|
40
40
|
- ext/deltalake/extconf.rb
|
41
41
|
- ext/deltalake/src/error.rs
|
42
|
+
- ext/deltalake/src/features.rs
|
42
43
|
- ext/deltalake/src/lib.rs
|
44
|
+
- ext/deltalake/src/merge.rs
|
43
45
|
- ext/deltalake/src/schema.rs
|
44
46
|
- ext/deltalake/src/utils.rs
|
45
47
|
- lib/deltalake-rb.rb
|
@@ -48,6 +50,10 @@ files:
|
|
48
50
|
- lib/deltalake/metadata.rb
|
49
51
|
- lib/deltalake/schema.rb
|
50
52
|
- lib/deltalake/table.rb
|
53
|
+
- lib/deltalake/table_alterer.rb
|
54
|
+
- lib/deltalake/table_merger.rb
|
55
|
+
- lib/deltalake/table_optimizer.rb
|
56
|
+
- lib/deltalake/utils.rb
|
51
57
|
- lib/deltalake/version.rb
|
52
58
|
homepage: https://github.com/ankane/delta-ruby
|
53
59
|
licenses:
|