rufus-tokyo 1.0.4 → 1.0.5

data/CHANGELOG.txt

@@ -2,10 +2,16 @@
 = rufus-tokyo CHANGELOG.txt
 
 
+== rufus-tokyo - 1.0.5    released 2010/01/06
+
+- todo  : now using getconf to determine INT_MIN (thanks oklahoma_mixer)
+- bug   : issue with ree and lib.free(pointer), switching to tcfree
+
+
 == rufus-tokyo - 1.0.4    released 2009/12/25
 
 - bug   : memory leak, gotten values not freed
-- todo  : Rufus::Tokyo::Tyrant and TyrantTable now reconnets (120 seconds)
+- todo  : Rufus::Tokyo::Tyrant and TyrantTable now reconnects (120 seconds)
 
 
 == rufus-tokyo - 1.0.3    released 2009/11/16

data/CREDITS.txt

@@ -25,6 +25,7 @@ Mark 'mebaran'  http://github.com/mebaran
 
 == feedback / support
 
+Aaron Zheng  free/tcfree issue
 Swerling  http://github.com/swerling
 Alexandre Gravem
 Philippe Monnet  http://blog.monnet-usa.com/

data/LICENSE.txt

@@ -1,5 +1,5 @@
 
-Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/cabcore.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/cabinet/abstract.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/cabinet/table.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/error.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/ntyrant/abstract.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/ntyrant/table.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/edo/tabcore.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -34,10 +34,14 @@ module Tokyo
   #
   class TokyoError < RuntimeError; end
 
+  # This getconf technique was taken from oklahoma_mixer :
+  # http://github.com/JEG2/oklahoma_mixer
   #
-  # Grumpf, this is not elegant...
-  #
-  INT_MIN = -2147483648
+  INT_MIN = if im = `getconf INT_MIN 2>&1`[/-\d+/]
+    im.to_i
+  else
+    -2147483648
+  end
 
   # Returns 'bytesize' of the string (Ruby 1.9.1 for everyone).
   #

data/lib/rufus/tokyo/cabinet/abstract.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/cabinet/lib.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/cabinet/table.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/cabinet/util.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/config.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/dystopia.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/dystopia/core.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, Jeremy Hinegardner, jeremy@copiousfreetime.org
+# Copyright (c) 2009-2010, Jeremy Hinegardner, jeremy@copiousfreetime.org
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/dystopia/lib.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, Jeremy Hinegardner, John Mettraux
+# Copyright (c) 2009-2010, Jeremy Hinegardner, John Mettraux
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/dystopia/words.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/hmethods.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/outlen.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, Adam Keys, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, Adam Keys, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -42,8 +42,8 @@ module Rufus::Tokyo
 
     ensure
       outlen.free
-      #lib.tcfree(out)
-      lib.free(out)
+      lib.tcfree(out)
+      #lib.free(out)
     end
   end
 end

data/lib/rufus/tokyo/query.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/transactions.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/ttcommons.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/tyrant.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/tyrant/abstract.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/tyrant/ext.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, Adam Keys, John Mettraux
+# Copyright (c) 2009-2010, Adam Keys, John Mettraux
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/tyrant/lib.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -54,6 +54,10 @@ module Rufus::Tokyo
       alias :attfunc :attach_function
     end
 
+    # frees a mem zone (TC style)
+    #
+    attfunc :tcfree, [ :pointer ], :void
+
     attfunc :free, [ :pointer ], :void
 
     # http://1978th.net/tokyotyrant/spex.html#tcrdbapi

data/lib/rufus/tokyo/tyrant/table.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/utils.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

data/lib/rufus/tokyo/version.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009-2010, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,7 @@
 module Rufus
 module Tokyo
 
-  VERSION = '1.0.4'
+  VERSION = '1.0.5'
 end
 end
 

data/rufus-tokyo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{rufus-tokyo}
-  s.version = "1.0.4"
+  s.version = "1.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John Mettraux", "Zev Blut", "Jeremy Hinegardner", "James Edward Gray II"]
-  s.date = %q{2009-12-25}
+  s.date = %q{2010-01-06}
   s.description = %q{
 Ruby-ffi based lib to access Tokyo Cabinet and Tyrant.
 
@@ -95,9 +95,11 @@ Finally rufus-tokyo includes ffi-based interfaces to Tokyo Dystopia (thanks to J
      "spec/util_list_spec.rb",
      "spec/util_map_spec.rb",
      "tasks/dev.rb",
+     "test/aaron.rb",
      "test/bm0.rb",
      "test/bm1_compression.rb",
      "test/con0.rb",
+     "test/jeg.rb",
      "test/mem.rb",
      "test/mem1.rb",
      "test/readme0.rb",

data/test/aaron.rb

@@ -0,0 +1,51 @@
+
+require 'rubygems'
+require 'rufus/tokyo'
+
+def show_memory
+ 3.times { GC.start }  # try to clean up
+ #mem = `ps -o rss -p #{Process.pid}`[/\d+/]
+ mem = `ps -o vsz -p #{Process.pid}`[/\d+/]
+ #mem = File.open("/proc/#{Process.pid}/status", 'r') { |ps|
+ #  14.times { ps.gets }
+ #  ps.gets.split[1].to_i
+ #}
+ puts "Current memory:  #{mem}"
+end
+
+p :before_put
+show_memory
+
+
+db = Rufus::Tokyo::Cabinet.new('test.tch')
+db['some_key'] = "X" * 1024
+
+p :after_put
+show_memory
+
+p :first_round
+
+10.times do
+  5000.times do
+    db["some_key"]  # reading causes the memory leak
+  end
+  show_memory
+end
+
+sleep 1
+show_memory
+
+p :second_round
+
+10.times do
+  5000.times do
+    db["some_key"]  # reading causes the memory leak
+  end
+  show_memory
+end
+
+db.close
+
+sleep 1
+show_memory
+

data/test/jeg.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby -wKU
+
+require "rubygems"
+require "ffi"
+
+# map the C interface
+module Lib
+ extend FFI::Library
+ ffi_lib(
+   *Array(
+     ENV.fetch(
+       "TOKYO_CABINET_LIB",
+       Dir["/{opt,usr}/{,local/}lib{,64}/libtokyocabinet.{dylib,so*}"]
+     )
+   )
+ )
+
+ attach_function :tcfree, [ :pointer ], :void
+
+ attach_function :tchdbnew,   [ ],                                  :pointer
+ attach_function :tchdbopen,  [:pointer, :string, :int],            :bool
+ attach_function :tchdbput,   [:pointer, :pointer, :int, :pointer,
+                               :int],                               :bool
+ attach_function :tchdbget,   [:pointer, :pointer, :int, :pointer], :pointer
+ attach_function :tchdbclose, [:pointer],                           :bool
+end
+
+# translate the interface to Ruby
+class TokyoCabinet
+ def self.open(*args)
+   db = new(*args)
+   yield db
+ ensure
+   db.close if db
+ end
+
+ def initialize(path)
+   @db = Lib.tchdbnew
+   Lib.tchdbopen(@db, path, (1 << 1) | (1 << 2))  # write create mode
+ end
+
+ def []=(key, value)
+   k, v = key.to_s, value.to_s
+   Lib.tchdbput(@db, k, k.size, v, v.size)
+ end
+
+ def [](key)
+   k     = key.to_s
+   size  = FFI::MemoryPointer.new(:int)
+   value = Lib.tchdbget(@db, k, k.size, size)
+   value.address.zero? ? nil : value.get_bytes(0, size.get_int(0))
+ ensure
+   size.free if size
+   # FIXME:  How do I free value here?
+   Lib.tcfree(value)
+ end
+
+ def close
+   Lib.tchdbclose(@db)
+ end
+end
+
+# show the problem
+def show_memory
+ 3.times { GC.start }  # try to clean up
+ mem = `ps -o rss -p #{Process.pid}`[/\d+/]
+ puts "Current memory:  #{mem}"
+end
+
+TokyoCabinet.open("leak.tch") do |db|
+ db[:some_key] = "X" * 1024
+ 10.times do
+   5000.times do
+     db[:some_key]  # reading causes the memory leak
+   end
+   show_memory
+ end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rufus-tokyo
 version: !ruby/object:Gem::Version 
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors: 
 - John Mettraux
@@ -12,7 +12,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-25 00:00:00 +09:00
+date: 2010-01-06 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -120,9 +120,11 @@ files:
 - spec/util_list_spec.rb
 - spec/util_map_spec.rb
 - tasks/dev.rb
+- test/aaron.rb
 - test/bm0.rb
 - test/bm1_compression.rb
 - test/con0.rb
+- test/jeg.rb
 - test/mem.rb
 - test/mem1.rb
 - test/readme0.rb