extlzham 0.0.1.PROTOTYPE → 0.0.1.PROTOTYPE2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 252b541fe39b87a810a1c32faf9f7a2f2ad57b11
-  data.tar.gz: f27f740cb6f08a39837c8fea305da68c63f784d7
+  metadata.gz: 38a6dfeed7afbd9178b072cbc682d5d239b396e5
+  data.tar.gz: a70c0b958bd0b1652830755f20ed86a2df3bc695
 SHA512:
-  metadata.gz: 780c3b5c764257695c861a72601c1d74ea92aa68e79059f4bd0d4a8c51dbe68cd978fe9a8d74e4fed9b28e1132d34a7c8457e553b5dc41aa124ff87050da084f
-  data.tar.gz: 82704ccee75a3a81480e328c29348fa71d27a9a3bc38c38d9c3ebc365064cda6f212dfe258b4731faf422e349324eb8e6c634e4247bb3378e3ea3b330f3e9533
+  metadata.gz: b2d6760152aac48b4e78646c02ea99fe1f5618a34bda0339de1c00ff955485650adfb389dfc729a02c03bf7d6b59e836f552923ae6810577b8a3acd5d5cebd26
+  data.tar.gz: 2a4324dd8844f739448e18f0f5654494a7e95d786cc6ee437c98142dd3e03a7179cabe1f402f9562dd260007efc826593c6d52f05540158bd953cb6c4c55d5d9

data/README.md

@@ -2,19 +2,72 @@
 
 # extlzham - ruby binding for LZHAM
 
-This is ruby binding for compress/decompress library
+This is ruby bindings for compression library
 [LZHAM (https://github.com/richgel999/lzham\_codec)](https://github.com/richgel999/lzham_codec).
 
 *   PACKAGE NAME: extlzham
 *   AUTHOR: dearblue <dearblue@users.sourceforge.jp>
-*   VERSION: 0.0.1.PROTOTYPE
+*   VERSION: 0.0.1.PROTOTYPE2
 *   LICENSING: 2-clause BSD License
 *   REPORT ISSUE TO: <http://sourceforge.jp/projects/rutsubo/ticket/>
 *   DEPENDENCY RUBY: ruby-2.0+
 *   DEPENDENCY RUBY GEMS: (none)
 *   DEPENDENCY LIBRARY: (none)
 *   BUNDLED EXTERNAL LIBRARIES:
-    *   LZHAM: https://github.com/richgel999/lzham\_codec
+    *   LZHAM <https://github.com/richgel999/lzham_codec>
+
+
+## HOW TO USE
+
+### Simply process
+
+``` ruby:ruby
+# First, load library
+require "extlzham"
+
+# Prepair data
+source = "sample data..." * 50
+
+# Directly compression
+encdata = LZHAM.encode(source)
+
+# Directly decompression
+decdata = LZHAM.decode(encdata)
+
+# Comparison source and decoded data
+p source == decdata
+```
+
+### Streaming process
+
+``` ruby:ruby
+# First, load library
+require "extlzham"
+
+# Prepair data
+source = "sample data..." * 50
+sourceio = StringIO.new(source)
+
+# streaming compression
+LZHAM.encode(dest = StringIO.new("")) do |encoder|
+  while buf = sourceio.read(50) # Please increase the value if you want to actually use.
+    encoder << buf
+  end
+  # block leave to encoder.close
+end
+
+# streaming decompression
+dest.rewind
+decdata = ""
+LZHAM.decode(StringIO.new(decdata)) do |decoder|
+  while buf = dest.read(50)
+    decoder << buf
+  end
+end
+
+# Comparison source and decoded data
+p source == decdata
+```
 
 ----
 

data/Rakefile

@@ -10,10 +10,11 @@ BIN = FileList["bin/*"]
 LIB = FileList["lib/**/*.rb"]
 SPEC = FileList["spec/**/*"]
 EXAMPLE = FileList["examples/**/*"]
-RAKEFILE = [File.basename(__FILE__), "gemstub.rb"]
+GEMSTUB_SRC = "gemstub.rb"
+RAKEFILE = [File.basename(__FILE__), GEMSTUB_SRC]
 EXTRA = []
 
-load "gemstub.rb"
+load GEMSTUB_SRC
 
 EXTCONF = FileList["ext/extconf.rb"]
 EXTCONF.reject! { |n| !File.file?(n) }
@@ -59,7 +60,7 @@ unless EXTCONF.empty?
     PLATFORM = platforms1[0]
 
     RUBY_VERSIONS = RUBYSET.map do |ruby|
-      ver = `#{ruby} --disable gem -rrbconfig -e "puts RbConfig::CONFIG['ruby_version']"`.chomp
+      ver = `#{ruby} --disable gem -rrbconfig -e "puts RbConfig::CONFIG['ruby_version']"`.slice(/\d+\.\d+/)
       raise "failed ruby checking - ``#{ruby}''" unless $?.success?
       [ver, ruby]
     end
@@ -85,10 +86,13 @@ unless EXTCONF.empty?
       sh "gem build #{GEMSPEC_NATIVE}"
     end
 
-    file GEMSPEC_NATIVE => __FILE__ do
+    file GEMSPEC_NATIVE => RAKEFILE do
       File.write(GEMSPEC_NATIVE, GEMSTUB_NATIVE.to_ruby, mode: "wb")
     end
 
+    desc "build c-extension libraries"
+    task "sofiles" => SOFILES
+
     SOFILES_SET.each do |(soname, ruby)|
       sodir = File.dirname(soname)
       makefile = File.join(sodir, "Makefile")

data/ext/constants.c

@@ -0,0 +1,50 @@
+#include "extlzham.h"
+
+VALUE mConsts;
+
+void
+init_constants(void)
+{
+    mConsts = rb_define_module_under(mLZHAM, "Constants");
+    rb_include_module(mLZHAM, mConsts);
+    rb_define_const(mConsts, "NO_FLUSH", INT2FIX(LZHAM_NO_FLUSH));
+    rb_define_const(mConsts, "SYNC_FLUSH", INT2FIX(LZHAM_SYNC_FLUSH));
+    rb_define_const(mConsts, "FULL_FLUSH", INT2FIX(LZHAM_FULL_FLUSH));
+    rb_define_const(mConsts, "FINISH", INT2FIX(LZHAM_FINISH));
+    rb_define_const(mConsts, "TABLE_FLUSH", INT2FIX(LZHAM_TABLE_FLUSH));
+    rb_define_const(mConsts, "MIN_DICT_SIZE_LOG2", INT2FIX(LZHAM_MIN_DICT_SIZE_LOG2));
+    rb_define_const(mConsts, "MAX_DICT_SIZE_LOG2_X86", INT2FIX(LZHAM_MAX_DICT_SIZE_LOG2_X86));
+    rb_define_const(mConsts, "MAX_DICT_SIZE_LOG2_X64", INT2FIX(LZHAM_MAX_DICT_SIZE_LOG2_X64));
+    rb_define_const(mConsts, "MAX_HELPER_THREADS", INT2FIX(LZHAM_MAX_HELPER_THREADS));
+    rb_define_const(mConsts, "COMP_FLAG_EXTREME_PARSING", INT2FIX(LZHAM_COMP_FLAG_EXTREME_PARSING));
+    rb_define_const(mConsts, "COMP_FLAG_DETERMINISTIC_PARSING", INT2FIX(LZHAM_COMP_FLAG_DETERMINISTIC_PARSING));
+    rb_define_const(mConsts, "COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO", INT2FIX(LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO));
+    rb_define_const(mConsts, "COMP_FLAG_WRITE_ZLIB_STREAM", INT2FIX(LZHAM_COMP_FLAG_WRITE_ZLIB_STREAM));
+    rb_define_const(mConsts, "INSANELY_SLOW_TABLE_UPDATE_RATE", INT2FIX(LZHAM_INSANELY_SLOW_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "SLOWEST_TABLE_UPDATE_RATE", INT2FIX(LZHAM_SLOWEST_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "DEFAULT_TABLE_UPDATE_RATE", INT2FIX(LZHAM_DEFAULT_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "FASTEST_TABLE_UPDATE_RATE", INT2FIX(LZHAM_FASTEST_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "DECOMP_FLAG_OUTPUT_UNBUFFERED", INT2FIX(LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED));
+    rb_define_const(mConsts, "DECOMP_FLAG_COMPUTE_ADLER32", INT2FIX(LZHAM_DECOMP_FLAG_COMPUTE_ADLER32));
+    rb_define_const(mConsts, "DECOMP_FLAG_READ_ZLIB_STREAM", INT2FIX(LZHAM_DECOMP_FLAG_READ_ZLIB_STREAM));
+    rb_define_const(mConsts, "LZHAM_NO_FLUSH", INT2FIX(LZHAM_NO_FLUSH));
+    rb_define_const(mConsts, "LZHAM_SYNC_FLUSH", INT2FIX(LZHAM_SYNC_FLUSH));
+    rb_define_const(mConsts, "LZHAM_FULL_FLUSH", INT2FIX(LZHAM_FULL_FLUSH));
+    rb_define_const(mConsts, "LZHAM_FINISH", INT2FIX(LZHAM_FINISH));
+    rb_define_const(mConsts, "LZHAM_TABLE_FLUSH", INT2FIX(LZHAM_TABLE_FLUSH));
+    rb_define_const(mConsts, "LZHAM_MIN_DICT_SIZE_LOG2", INT2FIX(LZHAM_MIN_DICT_SIZE_LOG2));
+    rb_define_const(mConsts, "LZHAM_MAX_DICT_SIZE_LOG2_X86", INT2FIX(LZHAM_MAX_DICT_SIZE_LOG2_X86));
+    rb_define_const(mConsts, "LZHAM_MAX_DICT_SIZE_LOG2_X64", INT2FIX(LZHAM_MAX_DICT_SIZE_LOG2_X64));
+    rb_define_const(mConsts, "LZHAM_MAX_HELPER_THREADS", INT2FIX(LZHAM_MAX_HELPER_THREADS));
+    rb_define_const(mConsts, "LZHAM_COMP_FLAG_EXTREME_PARSING", INT2FIX(LZHAM_COMP_FLAG_EXTREME_PARSING));
+    rb_define_const(mConsts, "LZHAM_COMP_FLAG_DETERMINISTIC_PARSING", INT2FIX(LZHAM_COMP_FLAG_DETERMINISTIC_PARSING));
+    rb_define_const(mConsts, "LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO", INT2FIX(LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO));
+    rb_define_const(mConsts, "LZHAM_COMP_FLAG_WRITE_ZLIB_STREAM", INT2FIX(LZHAM_COMP_FLAG_WRITE_ZLIB_STREAM));
+    rb_define_const(mConsts, "LZHAM_INSANELY_SLOW_TABLE_UPDATE_RATE", INT2FIX(LZHAM_INSANELY_SLOW_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "LZHAM_SLOWEST_TABLE_UPDATE_RATE", INT2FIX(LZHAM_SLOWEST_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "LZHAM_DEFAULT_TABLE_UPDATE_RATE", INT2FIX(LZHAM_DEFAULT_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "LZHAM_FASTEST_TABLE_UPDATE_RATE", INT2FIX(LZHAM_FASTEST_TABLE_UPDATE_RATE));
+    rb_define_const(mConsts, "LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED", INT2FIX(LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED));
+    rb_define_const(mConsts, "LZHAM_DECOMP_FLAG_COMPUTE_ADLER32", INT2FIX(LZHAM_DECOMP_FLAG_COMPUTE_ADLER32));
+    rb_define_const(mConsts, "LZHAM_DECOMP_FLAG_READ_ZLIB_STREAM", INT2FIX(LZHAM_DECOMP_FLAG_READ_ZLIB_STREAM));
+}

data/ext/decoder.c

@@ -0,0 +1,315 @@
+#include "extlzham.h"
+
+VALUE cDecoder;
+
+/*
+ * decoder
+ */
+
+static inline lzham_decompress_params
+aux_conv_decode_params(VALUE opts)
+{
+    lzham_decompress_params p;
+    memset(&p, 0, sizeof(p));
+    p.m_struct_size = sizeof(p);
+    if (NIL_P(opts)) {
+        p.m_dict_size_log2 = LZHAM_MIN_DICT_SIZE_LOG2;
+        p.m_table_update_rate = 0;
+        p.m_decompress_flags = 0; // (see lzham_decompress_flags enum)
+        p.m_num_seed_bytes = 0;
+        p.m_pSeed_bytes = NULL;
+        p.m_table_max_update_interval = 0;
+        p.m_table_update_interval_slow_rate = 0;
+    } else {
+        p.m_dict_size_log2 = aux_hash_lookup_to_u32(opts, IDdictsize, LZHAM_MIN_DICT_SIZE_LOG2);
+        p.m_table_update_rate = aux_hash_lookup_to_u32(opts, IDtable_update_rate, 0);
+        p.m_decompress_flags = aux_hash_lookup_to_u32(opts, IDflags, 0);
+        p.m_num_seed_bytes = 0;
+        p.m_pSeed_bytes = NULL;
+        p.m_table_max_update_interval = aux_hash_lookup_to_u32(opts, IDtable_max_update_interval, 0);
+        p.m_table_update_interval_slow_rate = aux_hash_lookup_to_u32(opts, IDtable_update_interval_slow_rate, 0);
+    }
+
+    return p;
+}
+
+/*
+ * call-seq:
+ *  decode(string, max_decoded_size, opts = {}) -> decoded string
+ */
+static VALUE
+ext_s_decode(int argc, VALUE argv[], VALUE mod)
+{
+    VALUE src, size, opts;
+    rb_scan_args(argc, argv, "2:", &src, &size, &opts);
+    rb_check_type(src, RUBY_T_STRING);
+    rb_str_locktmp(src);
+    size_t srcsize = RSTRING_LEN(src);
+    size_t destsize = NUM2SIZET(size);
+    VALUE dest = rb_str_buf_new(destsize);
+    lzham_decompress_params p = aux_conv_decode_params(opts);
+    lzham_decompress_status_t s;
+    s = lzham_decompress_memory(&p,
+            (lzham_uint8 *)RSTRING_PTR(dest), &destsize,
+            (lzham_uint8 *)RSTRING_PTR(src), srcsize, NULL);
+
+    rb_str_unlocktmp(src);
+
+    if (s != LZHAM_DECOMP_STATUS_SUCCESS) {
+        rb_str_resize(dest, 0);
+        aux_decode_error(s);
+    }
+
+    rb_str_resize(dest, destsize);
+    rb_str_set_len(dest, destsize);
+
+    return dest;
+}
+
+struct decoder
+{
+    lzham_decompress_state_ptr decoder;
+    VALUE outport;
+    VALUE outbuf;
+};
+
+static void
+ext_dec_mark(struct decoder *p)
+{
+    if (p) {
+        rb_gc_mark(p->outport);
+        rb_gc_mark(p->outbuf);
+    }
+}
+
+static void
+ext_dec_free(struct decoder *p)
+{
+    if (p) {
+        if (p->decoder) {
+            lzham_decompress_deinit(p->decoder);
+        }
+    }
+}
+
+static VALUE
+ext_dec_alloc(VALUE klass)
+{
+    struct decoder *p;
+    VALUE obj = Data_Make_Struct(klass, struct decoder, ext_dec_mark, ext_dec_free, p);
+    return obj;
+}
+
+static inline struct decoder *
+aux_decoder_refp(VALUE obj)
+{
+    struct decoder *p;
+    Data_Get_Struct(obj, struct decoder, p);
+    return p;
+}
+
+static inline struct decoder *
+aux_decoder_ref(VALUE obj)
+{
+    struct decoder *p = aux_decoder_refp(obj);
+    if (!p || !p->decoder) {
+        rb_raise(eError,
+                 "not initialized - #<%s:%p>",
+                 rb_obj_classname(obj), (void *)obj);
+    }
+    return p;
+}
+
+/*
+ * call-seq:
+ *  initialize(outport = nil, opts = {})
+ */
+static VALUE
+ext_dec_init(int argc, VALUE argv[], VALUE dec)
+{
+    struct decoder *p = DATA_PTR(dec);
+    if (p->decoder) {
+        rb_raise(eError,
+                 "already initialized - #<%s:%p>",
+                 rb_obj_classname(dec), (void *)dec);
+    }
+
+    VALUE outport, opts;
+    rb_scan_args(argc, argv, "01:", &outport, &opts);
+    if (NIL_P(outport)) { outport = rb_str_buf_new(0); }
+    lzham_decompress_params params = aux_conv_decode_params(opts);
+    p->decoder = lzham_decompress_init(&params);
+    if (!p->decoder) {
+        rb_raise(eError,
+                 "failed lzham_decompress_init - #<%s:%p>",
+                 rb_obj_classname(dec), (void *)dec);
+    }
+
+    p->outbuf = rb_str_buf_new(WORKBUF_SIZE);
+    p->outport = outport;
+
+    return dec;
+}
+
+struct aux_lzham_decompress_nogvl
+{
+    lzham_decompress_state_ptr state;
+    const lzham_uint8 *inbuf;
+    size_t *insize;
+    lzham_uint8 *outbuf;
+    size_t *outsize;
+    lzham_bool flags;
+};
+
+static void *
+aux_lzham_decompress_nogvl(void *px)
+{
+    struct aux_lzham_decompress_nogvl *p = px;
+    return (void *)lzham_decompress(p->state, p->inbuf, p->insize, p->outbuf, p->outsize, p->flags);
+}
+
+static inline lzham_decompress_status_t
+aux_lzham_decompress(lzham_decompress_state_ptr state,
+        const lzham_uint8 *inbuf, size_t *insize,
+        lzham_uint8 *outbuf, size_t *outsize,
+        lzham_bool flags)
+{
+    struct aux_lzham_decompress_nogvl p = {
+        .state = state,
+        .inbuf = inbuf,
+        .insize = insize,
+        .outbuf = outbuf,
+        .outsize = outsize,
+        .flags = flags,
+    };
+
+    return (lzham_decompress_status_t)rb_thread_call_without_gvl(aux_lzham_decompress_nogvl, &p, 0, 0);
+}
+
+struct dec_update_args
+{
+    VALUE decoder;
+    VALUE src;
+    int flush;
+};
+
+static VALUE
+dec_update_protected(struct dec_update_args *args)
+{
+    struct decoder *p = aux_decoder_ref(args->decoder);
+    const char *inbuf, *intail;
+
+    if (NIL_P(args->src)) {
+        inbuf = NULL;
+        intail = NULL;
+    } else {
+        inbuf = RSTRING_PTR(args->src);
+        intail = inbuf + RSTRING_LEN(args->src);
+    }
+
+    lzham_decompress_status_t s;
+    do {
+        size_t insize = intail - inbuf;
+        aux_str_reserve(p->outbuf, WORKBUF_SIZE);
+        rb_str_locktmp(p->outbuf);
+        size_t outsize = rb_str_capacity(p->outbuf);
+        s = aux_lzham_decompress(p->decoder,
+                (lzham_uint8 *)inbuf, &insize,
+                (lzham_uint8 *)RSTRING_PTR(p->outbuf), &outsize, args->flush);
+        rb_str_unlocktmp(p->outbuf);
+//fprintf(stderr, "%s:%d:%s: status=%s (%d), insize=%zu, outsize=%zu\n", __FILE__, __LINE__, __func__, aux_decode_status_str(s), s, insize, outsize);
+        if (!NIL_P(args->src)) {
+            inbuf += insize;
+        }
+        if (s != LZHAM_DECOMP_STATUS_NOT_FINISHED &&
+            s != LZHAM_DECOMP_STATUS_HAS_MORE_OUTPUT &&
+            s != LZHAM_DECOMP_STATUS_NEEDS_MORE_INPUT &&
+            s != LZHAM_DECOMP_STATUS_SUCCESS) {
+
+            aux_decode_error(s);
+        }
+        if (outsize > 0) {
+            rb_str_set_len(p->outbuf, outsize);
+            rb_funcall2(p->outport, ID_op_lshift, 1, &p->outbuf);
+        }
+    } while (inbuf < intail || s == LZHAM_DECOMP_STATUS_HAS_MORE_OUTPUT);
+
+    return 0;
+}
+
+static inline void
+dec_update(VALUE dec, VALUE src, int flush)
+{
+    struct dec_update_args args = { dec, src, flush };
+    if (NIL_P(src)) {
+        dec_update_protected(&args);
+    } else {
+        rb_str_locktmp(src);
+        int state;
+        rb_protect((VALUE (*)(VALUE))dec_update_protected, (VALUE)&args, &state);
+        rb_str_unlocktmp(src);
+        if (state) {
+            rb_jump_tag(state);
+        }
+    }
+}
+
+/*
+ * call-seq:
+ *  update(src, flush = false) -> self
+ */
+static VALUE
+ext_dec_update(int argc, VALUE argv[], VALUE dec)
+{
+    VALUE src, flush;
+    rb_scan_args(argc, argv, "11", &src, &flush);
+    rb_check_type(src, RUBY_T_STRING);
+    dec_update(dec, src, RTEST(flush) ? 1 : 0);
+    return dec;
+}
+
+static VALUE
+ext_dec_finish(VALUE dec)
+{
+    dec_update(dec, Qnil, 1);
+    return dec;
+}
+
+static VALUE
+ext_dec_op_lshift(VALUE dec, VALUE src)
+{
+    rb_check_type(src, RUBY_T_STRING);
+    dec_update(dec, src, 0);
+    return dec;
+}
+
+static VALUE
+ext_dec_get_outport(VALUE dec)
+{
+    struct decoder *p = aux_decoder_ref(dec);
+    return p->outport;
+}
+
+static VALUE
+ext_dec_set_outport(VALUE dec, VALUE outport)
+{
+    struct decoder *p = aux_decoder_ref(dec);
+    p->outport = outport;
+    return outport;
+}
+
+void
+init_decoder(void)
+{
+    cDecoder = rb_define_class_under(mLZHAM, "Decoder", rb_cObject);
+    rb_include_module(cDecoder, mConsts);
+    rb_define_alloc_func(cDecoder, ext_dec_alloc);
+    rb_define_singleton_method(cDecoder, "decode", RUBY_METHOD_FUNC(ext_s_decode), -1);
+    rb_define_method(cDecoder, "initialize", RUBY_METHOD_FUNC(ext_dec_init), -1);
+    rb_define_method(cDecoder, "update", RUBY_METHOD_FUNC(ext_dec_update), -1);
+    rb_define_method(cDecoder, "finish", RUBY_METHOD_FUNC(ext_dec_finish), 0);
+    rb_define_method(cDecoder, "<<", RUBY_METHOD_FUNC(ext_dec_op_lshift), 1);
+    rb_define_method(cDecoder, "outport", RUBY_METHOD_FUNC(ext_dec_get_outport), 0);
+    rb_define_method(cDecoder, "outport=", RUBY_METHOD_FUNC(ext_dec_set_outport), 1);
+
+}

data/ext/depend

@@ -0,0 +1,5 @@
+constants.o: constants.c extlzham.h
+decoder.o: decoder.c extlzham.h
+encoder.o: encoder.c extlzham.h
+error.o: error.c extlzham.h
+extlzham.o: extlzham.c extlzham.h