ruby-zstds 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/AUTHORS +1 -0
  3. data/LICENSE +21 -0
  4. data/README.md +498 -0
  5. data/ext/extconf.rb +82 -0
  6. data/ext/zstds_ext/buffer.c +30 -0
  7. data/ext/zstds_ext/buffer.h +23 -0
  8. data/ext/zstds_ext/common.h +16 -0
  9. data/ext/zstds_ext/dictionary.c +106 -0
  10. data/ext/zstds_ext/dictionary.h +16 -0
  11. data/ext/zstds_ext/error.c +81 -0
  12. data/ext/zstds_ext/error.h +35 -0
  13. data/ext/zstds_ext/io.c +512 -0
  14. data/ext/zstds_ext/io.h +14 -0
  15. data/ext/zstds_ext/macro.h +13 -0
  16. data/ext/zstds_ext/main.c +25 -0
  17. data/ext/zstds_ext/option.c +287 -0
  18. data/ext/zstds_ext/option.h +122 -0
  19. data/ext/zstds_ext/stream/compressor.c +241 -0
  20. data/ext/zstds_ext/stream/compressor.h +31 -0
  21. data/ext/zstds_ext/stream/decompressor.c +183 -0
  22. data/ext/zstds_ext/stream/decompressor.h +29 -0
  23. data/ext/zstds_ext/string.c +254 -0
  24. data/ext/zstds_ext/string.h +14 -0
  25. data/lib/zstds.rb +9 -0
  26. data/lib/zstds/dictionary.rb +47 -0
  27. data/lib/zstds/error.rb +22 -0
  28. data/lib/zstds/file.rb +46 -0
  29. data/lib/zstds/option.rb +194 -0
  30. data/lib/zstds/stream/abstract.rb +153 -0
  31. data/lib/zstds/stream/delegates.rb +36 -0
  32. data/lib/zstds/stream/raw/abstract.rb +55 -0
  33. data/lib/zstds/stream/raw/compressor.rb +101 -0
  34. data/lib/zstds/stream/raw/decompressor.rb +70 -0
  35. data/lib/zstds/stream/reader.rb +166 -0
  36. data/lib/zstds/stream/reader_helpers.rb +192 -0
  37. data/lib/zstds/stream/stat.rb +78 -0
  38. data/lib/zstds/stream/writer.rb +145 -0
  39. data/lib/zstds/stream/writer_helpers.rb +93 -0
  40. data/lib/zstds/string.rb +31 -0
  41. data/lib/zstds/validation.rb +48 -0
  42. data/lib/zstds/version.rb +6 -0
  43. metadata +182 -0
@@ -0,0 +1,153 @@
1
+ # Ruby bindings for zstd library.
2
+ # Copyright (c) 2019 AUTHORS, MIT License.
3
+
4
+ require_relative "delegates"
5
+ require_relative "stat"
6
+ require_relative "../error"
7
+ require_relative "../validation"
8
+
9
+ module ZSTDS
10
+ module Stream
11
+ class Abstract
12
+ # Native stream is not seekable by design.
13
+ # Related methods like "seek" and "pos=" can't be implemented.
14
+
15
+ # It is not possible to maintain correspondance between bytes consumed from source and bytes written to destination by design.
16
+ # We will consume all source bytes and maintain buffer with remaining destination data.
17
+
18
+ include Delegates
19
+
20
+ attr_reader :io
21
+ attr_reader :stat
22
+ attr_reader :external_encoding
23
+ attr_reader :internal_encoding
24
+ attr_reader :transcode_options
25
+ attr_reader :pos
26
+ alias tell pos
27
+
28
+ def initialize(io, external_encoding: nil, internal_encoding: nil, transcode_options: {})
29
+ @raw_stream = create_raw_stream
30
+
31
+ Validation.validate_io io
32
+ @io = io
33
+
34
+ @stat = Stat.new @io.stat
35
+
36
+ set_encoding external_encoding, internal_encoding, transcode_options
37
+ reset_buffer
38
+ reset_io_advise
39
+
40
+ @pos = 0
41
+ end
42
+
43
+ # -- buffer --
44
+
45
+ protected def reset_buffer
46
+ @buffer = ::String.new :encoding => ::Encoding::BINARY
47
+ end
48
+
49
+ # -- advise --
50
+
51
+ protected def reset_io_advise
52
+ # Both compressor and decompressor need sequential io access.
53
+ @io.advise :sequential
54
+ rescue ::Errno::ESPIPE # rubocop:disable Lint/HandleExceptions
55
+ # ok
56
+ end
57
+
58
+ def advise
59
+ # Noop
60
+ nil
61
+ end
62
+
63
+ # -- encoding --
64
+
65
+ def set_encoding(*args)
66
+ external_encoding, internal_encoding, transcode_options = process_set_encoding_arguments(*args)
67
+
68
+ set_target_encoding :@external_encoding, external_encoding
69
+ set_target_encoding :@internal_encoding, internal_encoding
70
+ @transcode_options = transcode_options
71
+
72
+ self
73
+ end
74
+
75
+ protected def process_set_encoding_arguments(*args)
76
+ external_encoding = args[0]
77
+
78
+ unless external_encoding.nil? || external_encoding.is_a?(::Encoding)
79
+ Validation.validate_string external_encoding
80
+
81
+ # First argument can be "external_encoding:internal_encoding".
82
+ match = %r{(.+?):(.+)}.match external_encoding
83
+
84
+ unless match.nil?
85
+ external_encoding = match[0]
86
+ internal_encoding = match[1]
87
+
88
+ transcode_options = args[1]
89
+ Validation.validate_hash transcode_options unless transcode_options.nil?
90
+
91
+ return [external_encoding, internal_encoding, transcode_options]
92
+ end
93
+ end
94
+
95
+ internal_encoding = args[1]
96
+ Validation.validate_string internal_encoding \
97
+ unless internal_encoding.nil? || internal_encoding.is_a?(::Encoding)
98
+
99
+ transcode_options = args[2]
100
+ Validation.validate_hash transcode_options unless transcode_options.nil?
101
+
102
+ [external_encoding, internal_encoding, transcode_options]
103
+ end
104
+
105
+ protected def set_target_encoding(name, value)
106
+ unless value.nil? || value.is_a?(::Encoding)
107
+ begin
108
+ value = ::Encoding.find value
109
+ rescue ::ArgumentError
110
+ raise ValidateError, "invalid #{name} encoding"
111
+ end
112
+ end
113
+
114
+ instance_variable_set name, value
115
+ end
116
+
117
+ protected def target_encoding
118
+ return @internal_encoding unless @internal_encoding.nil?
119
+ return @external_encoding unless @external_encoding.nil?
120
+
121
+ ::Encoding::BINARY
122
+ end
123
+
124
+ # -- etc --
125
+
126
+ def rewind
127
+ @raw_stream = create_raw_stream
128
+
129
+ @io.rewind
130
+ reset_buffer
131
+ reset_io_advise
132
+
133
+ @pos = 0
134
+
135
+ 0
136
+ end
137
+
138
+ def close
139
+ @io.close
140
+
141
+ nil
142
+ end
143
+
144
+ def closed?
145
+ @raw_stream.closed? && @io.closed?
146
+ end
147
+
148
+ def to_io
149
+ self
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,36 @@
1
+ # Ruby bindings for zstd library.
2
+ # Copyright (c) 2019 AUTHORS, MIT License.
3
+
4
+ require "forwardable"
5
+
6
+ module ZSTDS
7
+ module Stream
8
+ module Delegates
9
+ DELEGATES = %i[
10
+ autoclose=
11
+ autoclose?
12
+ binmode
13
+ binmode?
14
+ close_on_exec=
15
+ close_on_exec?
16
+ fcntl
17
+ fdatasync
18
+ fileno
19
+ fsync
20
+ ioctl
21
+ isatty
22
+ pid
23
+ sync
24
+ sync=
25
+ to_i
26
+ tty?
27
+ ]
28
+ .freeze
29
+
30
+ def self.included(klass)
31
+ klass.extend ::Forwardable
32
+ klass.def_delegators :@io, *DELEGATES
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,55 @@
1
+ # Ruby bindings for zstd library.
2
+ # Copyright (c) 2019 AUTHORS, MIT License.
3
+
4
+ require "zstds_ext"
5
+
6
+ require_relative "../../error"
7
+ require_relative "../../validation"
8
+
9
+ module ZSTDS
10
+ module Stream
11
+ module Raw
12
+ class Abstract
13
+ def initialize(native_stream)
14
+ @native_stream = native_stream
15
+ @is_closed = false
16
+ end
17
+
18
+ # -- write --
19
+
20
+ def flush(&writer)
21
+ write_result(&writer)
22
+ end
23
+
24
+ protected def flush_destination_buffer(&writer)
25
+ result_bytesize = write_result(&writer)
26
+ raise NotEnoughDestinationError, "not enough destination" if result_bytesize == 0
27
+ end
28
+
29
+ protected def write_result(&_writer)
30
+ result = @native_stream.read_result
31
+ yield result
32
+
33
+ result.bytesize
34
+ end
35
+
36
+ # -- close --
37
+
38
+ protected def do_not_use_after_close
39
+ raise UsedAfterCloseError, "used after close" if closed?
40
+ end
41
+
42
+ def close(&writer)
43
+ write_result(&writer)
44
+
45
+ @native_stream.close
46
+ @is_closed = true
47
+ end
48
+
49
+ def closed?
50
+ @is_closed
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,101 @@
1
+ # Ruby bindings for zstd library.
2
+ # Copyright (c) 2019 AUTHORS, MIT License.
3
+
4
+ require "zstds_ext"
5
+
6
+ require_relative "abstract"
7
+ require_relative "../../error"
8
+ require_relative "../../option"
9
+ require_relative "../../validation"
10
+
11
+ module ZSTDS
12
+ module Stream
13
+ module Raw
14
+ class Compressor < Abstract
15
+ BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
16
+
17
+ def initialize(options = {})
18
+ options = Option.get_compressor_options options, BUFFER_LENGTH_NAMES
19
+
20
+ pledged_size = options[:pledged_size]
21
+ Validation.validate_not_negative_integer pledged_size unless pledged_size.nil?
22
+
23
+ native_stream = NativeCompressor.new options
24
+
25
+ super native_stream
26
+ end
27
+
28
+ def write(source, &writer)
29
+ do_not_use_after_close
30
+
31
+ Validation.validate_string source
32
+ Validation.validate_proc writer
33
+
34
+ total_bytes_written = 0
35
+
36
+ loop do
37
+ bytes_written, need_more_destination = @native_stream.write source
38
+ total_bytes_written += bytes_written
39
+
40
+ if need_more_destination
41
+ source = source.byteslice bytes_written, source.bytesize - bytes_written
42
+ flush_destination_buffer(&writer)
43
+ next
44
+ end
45
+
46
+ unless bytes_written == source.bytesize
47
+ # Compressor write should eat all provided "source" without remainder.
48
+ raise UnexpectedError, "unexpected error"
49
+ end
50
+
51
+ break
52
+ end
53
+
54
+ total_bytes_written
55
+ end
56
+
57
+ def flush(&writer)
58
+ do_not_use_after_close
59
+
60
+ Validation.validate_proc writer
61
+
62
+ loop do
63
+ need_more_destination = @native_stream.flush
64
+
65
+ if need_more_destination
66
+ flush_destination_buffer(&writer)
67
+ next
68
+ end
69
+
70
+ break
71
+ end
72
+
73
+ super
74
+
75
+ nil
76
+ end
77
+
78
+ def close(&writer)
79
+ return nil if closed?
80
+
81
+ Validation.validate_proc writer
82
+
83
+ loop do
84
+ need_more_destination = @native_stream.finish
85
+
86
+ if need_more_destination
87
+ flush_destination_buffer(&writer)
88
+ next
89
+ end
90
+
91
+ break
92
+ end
93
+
94
+ super
95
+
96
+ nil
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,70 @@
1
+ # Ruby bindings for zstd library.
2
+ # Copyright (c) 2019 AUTHORS, MIT License.
3
+
4
+ require "zstds_ext"
5
+
6
+ require_relative "abstract"
7
+ require_relative "../../option"
8
+ require_relative "../../validation"
9
+
10
+ module ZSTDS
11
+ module Stream
12
+ module Raw
13
+ class Decompressor < Abstract
14
+ BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
15
+
16
+ def initialize(options = {})
17
+ options = Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
18
+ native_stream = NativeDecompressor.new options
19
+
20
+ super native_stream
21
+ end
22
+
23
+ def read(source, &writer)
24
+ do_not_use_after_close
25
+
26
+ Validation.validate_string source
27
+ Validation.validate_proc writer
28
+
29
+ total_bytes_read = 0
30
+
31
+ loop do
32
+ bytes_read, need_more_destination = @native_stream.read source
33
+ total_bytes_read += bytes_read
34
+
35
+ if need_more_destination
36
+ source = source.byteslice bytes_read, source.bytesize - bytes_read
37
+ flush_destination_buffer(&writer)
38
+ next
39
+ end
40
+
41
+ break
42
+ end
43
+
44
+ # Please remember that "total_bytes_read" can not be equal to "source.bytesize".
45
+ total_bytes_read
46
+ end
47
+
48
+ def flush(&writer)
49
+ do_not_use_after_close
50
+
51
+ Validation.validate_proc writer
52
+
53
+ super
54
+
55
+ nil
56
+ end
57
+
58
+ def close(&writer)
59
+ return nil if closed?
60
+
61
+ Validation.validate_proc writer
62
+
63
+ super
64
+
65
+ nil
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,166 @@
1
+ # Ruby bindings for zstd library.
2
+ # Copyright (c) 2019 AUTHORS, MIT License.
3
+
4
+ require_relative "abstract"
5
+ require_relative "raw/decompressor"
6
+ require_relative "reader_helpers"
7
+ require_relative "../validation"
8
+
9
+ module ZSTDS
10
+ module Stream
11
+ class Reader < Abstract
12
+ include ReaderHelpers
13
+
14
+ attr_accessor :lineno
15
+
16
+ def initialize(source_io, options = {}, *args)
17
+ @options = options
18
+
19
+ super source_io, *args
20
+
21
+ initialize_source_buffer_length
22
+ reset_io_remainder
23
+
24
+ @lineno = 0
25
+ end
26
+
27
+ protected def initialize_source_buffer_length
28
+ source_buffer_length = @options[:source_buffer_length]
29
+ Validation.validate_not_negative_integer source_buffer_length unless source_buffer_length.nil?
30
+
31
+ source_buffer_length = Buffer::DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR \
32
+ if source_buffer_length == 0 || source_buffer_length.nil?
33
+
34
+ @source_buffer_length = source_buffer_length
35
+ end
36
+
37
+ protected def create_raw_stream
38
+ Raw::Decompressor.new @options
39
+ end
40
+
41
+ protected def reset_io_remainder
42
+ @io_remainder = ::String.new :encoding => ::Encoding::BINARY
43
+ end
44
+
45
+ # -- synchronous --
46
+
47
+ def read(bytes_to_read = nil, out_buffer = nil)
48
+ Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
49
+ Validation.validate_string out_buffer unless out_buffer.nil?
50
+
51
+ return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read == 0
52
+
53
+ unless bytes_to_read.nil?
54
+ return nil if eof?
55
+
56
+ read_more_from_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
57
+
58
+ return read_bytes_from_buffer bytes_to_read, out_buffer
59
+ end
60
+
61
+ read_more_from_buffer until @io.eof?
62
+
63
+ result = @buffer
64
+ reset_buffer
65
+ @pos += result.bytesize
66
+
67
+ result.force_encoding @external_encoding unless @external_encoding.nil?
68
+ result = transcode_to_internal result
69
+ result = out_buffer.replace result unless out_buffer.nil?
70
+
71
+ result
72
+ end
73
+
74
+ protected def read_more_from_buffer
75
+ io_data = @io.read @source_buffer_length
76
+ append_io_data_to_buffer io_data
77
+ end
78
+
79
+ def readpartial(bytes_to_read = nil, out_buffer = nil)
80
+ raise ::EOFError if eof?
81
+
82
+ readpartial_from_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
83
+
84
+ read_bytes_from_buffer bytes_to_read, out_buffer
85
+ end
86
+
87
+ protected def readpartial_from_buffer
88
+ io_data = @io.readpartial @source_buffer_length
89
+ append_io_data_to_buffer io_data
90
+ end
91
+
92
+ def rewind
93
+ raw_wrapper :close
94
+
95
+ reset_io_remainder
96
+
97
+ super
98
+ end
99
+
100
+ def close
101
+ raw_wrapper :close
102
+
103
+ super
104
+ end
105
+
106
+ # -- asynchronous --
107
+
108
+ def read_nonblock(bytes_to_read, out_buffer = nil, *options)
109
+ raise ::EOFError if eof?
110
+
111
+ read_more_from_buffer_nonblock(*options) until @buffer.bytesize >= bytes_to_read || @io.eof?
112
+
113
+ read_bytes_from_buffer bytes_to_read, out_buffer
114
+ end
115
+
116
+ protected def read_more_from_buffer_nonblock(*options)
117
+ io_data = @io.read_nonblock @source_buffer_length, *options
118
+ append_io_data_to_buffer io_data
119
+ end
120
+
121
+ # -- common --
122
+
123
+ def eof?
124
+ @io.eof? && @buffer.bytesize == 0
125
+ end
126
+
127
+ protected def read_bytes_from_buffer(bytes_to_read, out_buffer)
128
+ bytes_read = [@buffer.bytesize, bytes_to_read].min
129
+
130
+ # Result uses buffer binary encoding.
131
+ result = @buffer.byteslice 0, bytes_read
132
+ @buffer = @buffer.byteslice bytes_read, @buffer.bytesize - bytes_read
133
+ @pos += bytes_read
134
+
135
+ result = out_buffer.replace result unless out_buffer.nil?
136
+ result
137
+ end
138
+
139
+ protected def append_io_data_to_buffer(io_data)
140
+ io_portion = @io_remainder + io_data
141
+ bytes_read = raw_wrapper :read, io_portion
142
+ @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
143
+
144
+ # We should just ignore case when "io.eof?" appears but "io_remainder" is not empty.
145
+ # Ancient compress implementations can write bytes from not initialized buffer parts to output.
146
+ raw_wrapper :flush if @io.eof?
147
+ end
148
+
149
+ protected def transcode_to_internal(data)
150
+ data = data.encode @internal_encoding, @transcode_options unless @internal_encoding.nil?
151
+ data
152
+ end
153
+
154
+ # We should be able to return data back to buffer.
155
+ # We won't use any transcode options because transcoded data should be backward compatible.
156
+ protected def transcode_to_external(data)
157
+ data = data.encode @external_encoding unless @external_encoding.nil?
158
+ data
159
+ end
160
+
161
+ protected def raw_wrapper(method_name, *args)
162
+ @raw_stream.send(method_name, *args) { |portion| @buffer << portion }
163
+ end
164
+ end
165
+ end
166
+ end