hmsearch-postgres 0.1.5 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72fb908dd77f9508a65f066ffc74fe758d0a96a8
-  data.tar.gz: 809da412d1dd4f575a8152bd1ce539656b1f8bca
+  metadata.gz: 0ebd6e4d5fd861317d794ccfd7fc135ae815c157
+  data.tar.gz: 08f0655edbc17a90762046442c155b643e36ced6
 SHA512:
-  metadata.gz: 3b0035a08eb6f1861a314338145fcd066d931a8f01b971f4a0296308ac70d7b4cfe526cfb92e1195b8060ab3fa6ef63d719baf100b8c18b625368f0af9c95df7
-  data.tar.gz: e011b8ee60b9ed328889507cb6c7dd04732455c5365a3a7e9fdbe4d131ae90002b5c51fae5ebbaed4764b5e0ef4795a00e68912233dfa6828da6a6e892e9d1fd
+  metadata.gz: 081e81c4357ff8ffc56ed28033c7f83319b1a981e7a6b754700b9c52ffc923e988dec484e832df7d0ea6a6429eb22b5968cd4a32e3ba75dfe5d8a39577ed656e
+  data.tar.gz: af5f422032d83d3e9437f5942aeb60f4e81355d0fc4698d22f2af5945f2204367af6566664e438623718551ed1cbc33ad5c409c4768af7804489989d2020df94

data/.gitignore

@@ -1,3 +1,3 @@
 Gemfile.lock
-tmp
+build
 lib/hmsearch/postgres_ext.*

data/.travis.yml

@@ -1,13 +1,3 @@
 language: ruby
 rvm:
   - 2.1.5
-before_install:
-  - mkdir -p $TRAVIS_BUILD_DIR/vendor
-  - curl -O http://pqxx.org/download/software/libpqxx/libpqxx-4.0.1.tar.gz
-  - tar -xvf libpqxx-4.0.1.tar.gz
-  - cd libpqxx-4.0.1
-  - ./configure --prefix=$TRAVIS_BUILD_DIR/vendor
-  - make
-  - make install
-  - cd ..
-  - bundle config --local build.hmsearch-postgres -- --with-pqxx-include=$TRAVIS_BUILD_DIR/vendor/include\ --with-pqxx-lib=$TRAVIS_BUILD_DIR/vendor/lib

data/README.md

@@ -0,0 +1,4 @@
+hmsearch-postgres gem
+=====================
+
+[![Build Status](https://travis-ci.org/denisnazarov/hmsearch-postgres-ruby.svg?branch=master)](https://travis-ci.org/denisnazarov/hmsearch-postgres-ruby)

data/Rakefile

@@ -1,26 +1,65 @@
 require 'rake/clean'
+require 'rbconfig'
 
-CLEAN.include('lib/hmsearch/Makefile')
+$root_dir = File.expand_path('..', __FILE__)
+$build_dir = File.join($root_dir, 'build')
+$vendor_dir = File.join($build_dir, 'vendor')
+$lib_dir = File.join($root_dir, 'lib/hmsearch')
+$extconf = File.join($root_dir, 'ext/hmsearch/extconf.rb')
+$libpqxx_url = "http://pqxx.org/download/software/libpqxx/libpqxx-4.0.1.tar.gz"
+$libpqxx_tar = "#{$build_dir}/libpqxx-4.0.1.tar.gz"
+$libpqxx_dir = "#{$build_dir}/libpqxx-4.0.1"
+$libpqxx_header = "#{$vendor_dir}/include/pqxx/pqxx"
+$libpqxx_include = "#{$vendor_dir}/include"
+$libpqxx_lib = "#{$vendor_dir}/lib"
+$dlext = RbConfig::CONFIG['DLEXT']
+$build_ext_dylib = "#{$build_dir}/postgres_ext.#{$dlext}"
+$lib_ext_dylib = "#{$lib_dir}/postgres_ext.#{$dlext}"
 
-CLOBBER.include('lib/hmsearch/postgres_ext.*')
+CLEAN.include($build_dir)
+CLOBBER.include($lib_ext_dylib)
 
 task default: :test
 
-desc 'build native extension'
-task :build do
-  mkdir_p 'tmp'
-  chdir 'tmp' do
-    sh 'ruby ../ext/hmsearch/extconf.rb'
+directory $build_dir
+
+directory $vendor_dir
+
+file $libpqxx_tar => $build_dir do
+  chdir $build_dir do
+    sh "curl -O #{$libpqxx_url}"
+  end
+end
+
+file $libpqxx_dir => $libpqxx_tar do
+  chdir $build_dir do
+    sh 'tar xf libpqxx-4.0.1.tar.gz'
+  end
+end
+
+file $libpqxx_header => [$vendor_dir,  $libpqxx_dir] do
+  chdir $libpqxx_dir do
+    sh "./configure --enable-shared --prefix=#{$vendor_dir}"
     sh 'make'
+    sh 'make install'
   end
+end
 
-  require 'rbconfig'
-  dlext = RbConfig::CONFIG['DLEXT']
-  cp "tmp/postgres_ext.#{dlext}", "lib/hmsearch/"
+file $build_ext_dylib => $libpqxx_header do
+  chdir $build_dir do
+    sh "ruby ../ext/hmsearch/extconf.rb -- --with-pqxx-include=#{$libpqxx_include}\ --with-pqxx-lib=#{$libpqxx_lib}"
+    sh 'make'
+  end
 end
 
+file $lib_ext_dylib => $build_ext_dylib do
+  cp $build_ext_dylib, $lib_ext_dylib
+end
+
+task build: $lib_ext_dylib
+
 desc 'run tests'
-task test: :build do
+task test: $lib_ext_dylib do
   $:.unshift('lib')
   FileList.new('./test/*_test.rb').each do |test|
     load test

data/ext/hmsearch/extconf.rb

@@ -5,14 +5,7 @@ dir_config('pqxx')
 
 have_library('pqxx')
 
-with_cppflags('--std=c++11') do
-	try_cpp(<<-CPP)
-		#include <string>
-		int main() {
-			std::to_string(0);
-			return 0;
-		}
-	CPP
-end
+$CPPFLAGS += ' --std=c++0x'
+$LDFLAGS += ' --std=c++0x'
 
 create_makefile("hmsearch/postgres_ext")

data/ext/hmsearch/hmsearch.cc

@@ -38,6 +38,18 @@
 
 #include "hmsearch.h"
 
+#include <sstream>
+
+namespace patch
+{
+  template < typename T > std::string to_string( const T& n )
+  {
+    std::ostringstream stm ;
+    stm << n ;
+    return stm.str() ;
+  }
+}
+
 /** The actual implementation of the HmSearch database.
  *
  * A difference between this implementation and the HmSearch algorithm
@@ -74,24 +86,24 @@ public:
     {
         _db = new pqxx::connection(connstr);
     }
-    
+
     ~HmSearchImpl() {
         close();
     }
-    
+
     bool insert(const hash_string& hash,
                 std::string* error_msg = NULL);
-    
+
     bool print_copystring(const hash_string& hash,
                           std::string* error_msg = NULL);
-    
+
     bool lookup(const hash_string& query,
                 LookupResultList& result,
                 int max_error = -1,
                 std::string* error_msg = NULL);
-    
+
     bool close(std::string* error_msg = NULL);
-    
+
 private:
     struct Candidate {
         Candidate() : matches(0), first_match(0), second_match(0) {}
@@ -99,18 +111,18 @@ private:
         int first_match;
         int second_match;
     };
-    
+
     typedef std::map<hash_string, Candidate> CandidateMap;
-    
+
     void get_candidates(const hash_string& query, CandidateMap& candidates);
     void add_hash_candidates(CandidateMap& candidates, int match,
                              const uint8_t* hashes, size_t length);
     hash_string get_multiple_keys(uint8_t *key, int partition);
     bool valid_candidate(const Candidate& candidate);
     int hamming_distance(const hash_string& query, const hash_string& hash);
-    
+
     int get_partition_key(const hash_string& hash, int partition, uint8_t *key);
-    
+
     pqxx::connection *_db;
     int _hash_bits;
     int _max_error;
@@ -118,7 +130,7 @@ private:
     int _partitions;
     int _partition_bits;
     int _partition_bytes;
-    
+
     static int one_bits[256];
 };
 
@@ -133,35 +145,35 @@ bool HmSearch::init(const std::string& path,
         error_msg = &dummy;
     }
     *error_msg = "";
-    
+
     if (hash_bits == 0 || (hash_bits & 7)) {
         *error_msg = "invalid hash_bits value";
         return false;
     }
-    
+
     if (max_error == 0 || max_error >= hash_bits || max_error > 518) {
         *error_msg = "invalid max_error value";
         return false;
     }
-    
+
     pqxx::connection db(path);
     if (!db.is_open()) {
         *error_msg = "Can't open database";
         return false;
     }
-    
+
     std::string sql;
     sql = "INSERT INTO config VALUES ($1, $2)";
     db.prepare("hash_max", sql);
-    
+
     pqxx::work W(db);
     sql = "CREATE TABLE IF NOT EXISTS config ("\
     "       hash_bits int,"\
     "       max_error int); TRUNCATE config";
     W.exec(sql);
-    
+
     W.prepared("hash_max")(hash_bits)(max_error).exec();
-    
+
     for (unsigned int i = 0; i < ((max_error + 3) / 2); i++) {
         {
             std::stringstream s;
@@ -170,24 +182,24 @@ bool HmSearch::init(const std::string& path,
             << "       key bytea); TRUNCATE partition" << i;
             W.exec(s.str());
         }
-        
+
         {
             std::stringstream s;
             s << "DROP INDEX IF EXISTS ix_key_" << i;
             W.exec(s.str());
         }
-        
+
         {
             std::stringstream s;
             s << "CREATE INDEX ix_key_" << i << " ON partition" << i << "(key)";
             W.exec(s.str());
         }
     }
-    
+
     W.commit();
-    
+
     db.disconnect();
-    
+
     return true;
 }
 
@@ -196,37 +208,37 @@ HmSearch* HmSearch::open(const std::string& path,
                          std::string* error_msg)
 {
     std::string dummy;
-    
+
     if (!error_msg) {
         error_msg = &dummy;
     }
     *error_msg = "";
-    
+
     try {
         pqxx::connection db(path);
-        
+
         std::string sql;
-        
+
         sql = "SELECT max_error, hash_bits FROM config";
         pqxx::nontransaction n(db);
         pqxx::result res(n.exec(sql));
-        
+
         pqxx::result::const_iterator c = res.begin(); // We retrieve just one row
-        
+
         unsigned long hash_bits, max_error;
         max_error = c[0].as<long>();
         hash_bits = c[1].as<long>();
-        
+
         db.disconnect();
-        
+
         HmSearch* hm = new HmSearchImpl(path, (int)hash_bits, (int)max_error);
         if (!hm) {
             *error_msg = "out of memory";
             return NULL;
         }
-        
+
         return hm;
-        
+
     }
     catch (const pqxx::broken_connection& e) {
         *error_msg = e.what();
@@ -239,33 +251,33 @@ HmSearch::hash_string HmSearch::parse_hexhash(const std::string& hexhash)
 {
     int len = (int)hexhash.length() / 2;
     uint8_t hash[len];
-    
+
     for (int i = 0; i < len; i++) {
         char buf[3];
         char* err;
-        
+
         buf[0] = hexhash[i * 2];
         buf[1] = hexhash[i * 2 + 1];
         buf[2] = 0;
-        
+
         hash[i] = strtoul(buf, &err, 16);
-        
+
         if (*err != '\0') {
             return hash_string();
         }
     }
-    
+
     return hash_string(hash, len);
 }
 
 std::string HmSearch::format_hexhash(const HmSearch::hash_string& hash)
 {
     char hex[hash.length() * 2 + 1];
-    
+
     for (size_t i = 0; i < hash.length(); i++) {
         sprintf(hex + 2 * i, "%02x", hash[i]);
     }
-    
+
     return hex;
 }
 
@@ -278,26 +290,26 @@ bool HmSearchImpl::print_copystring(const hash_string& hash,
         error_msg = &dummy;
     }
     *error_msg = "";
-    
+
     if (hash.length() != (size_t) _hash_bytes) {
         *error_msg = "incorrect hash length";
         return false;
     }
-    
+
     for (int i = 0; i < _partitions; i++) {
         uint8_t key[_partition_bytes];
-        
+
         get_partition_key(hash, i, key);
-        
+
         std::cout << "\\\\x" << format_hexhash(hash)
         << " "
         << int(i)
         << " "
         << "\\\\x" << format_hexhash(hash_string(key, _partition_bytes))
         << std::endl;
-        
+
     }
-    
+
     return true;
 }
 
@@ -309,37 +321,37 @@ bool HmSearchImpl::insert(const hash_string& hash,
         error_msg = &dummy;
     }
     *error_msg = "";
-    
+
     if (hash.length() != (size_t) _hash_bytes) {
         *error_msg = "incorrect hash length";
         return false;
     }
-    
+
     if (!_db->is_open()) {
         *error_msg = "database is closed";
         return false;
     }
-    
+
     for (int i = 0; i < _partitions; i++) {
         std::stringstream s;
         s << "INSERT INTO partition" << i
         << "  VALUES ($1, $2)";
-        _db->prepare("insert_"+std::to_string(i), s.str());
+        _db->prepare("insert_"+patch::to_string(static_cast<long long>(i)), s.str());
     }
     pqxx::work W(*_db);
     for (int i = 0; i < _partitions; i++) {
         uint8_t key[_partition_bytes];
-        
+
         get_partition_key(hash, i, key);
-        
+
         pqxx::binarystring key_blob(key, _partition_bytes);
         pqxx::binarystring hash_blob(hash.data(), hash.length());
-        
-        W.prepared("insert_"+std::to_string(i))(hash_blob)(key_blob).exec();
+
+        W.prepared("insert_"+patch::to_string(static_cast<long long>(i)))(hash_blob)(key_blob).exec();
     }
-    
+
     W.commit();
-    
+
     return true;
 }
 
@@ -354,25 +366,25 @@ bool HmSearchImpl::lookup(const hash_string& query,
         error_msg = &dummy;
     }
     *error_msg = "";
-    
+
     if (query.length() != (size_t) _hash_bytes) {
         *error_msg = "incorrect hash length";
         return false;
     }
-    
+
     if (!_db->is_open()) {
         *error_msg = "database is closed";
         return false;
     }
-    
+
     try {
         CandidateMap candidates;
         get_candidates(query, candidates);
-        
+
         for (CandidateMap::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
             if (valid_candidate(i->second)) {
                 int distance = hamming_distance(query, i->first);
-                
+
                 if (distance <= _max_error
                     && (reduced_error < 0 || distance <= reduced_error)) {
                     result.push_back(LookupResult(i->first, distance));
@@ -384,7 +396,7 @@ bool HmSearchImpl::lookup(const hash_string& query,
         *error_msg = e.base().what();
         return false;
     }
-    
+
     return true;
 }
 
@@ -396,14 +408,14 @@ bool HmSearchImpl::close(std::string* error_msg)
         error_msg = &dummy;
     }
     *error_msg = "";
-    
+
     if (!_db->is_open()) {
         // Already closed
         return true;
     }
-    
+
     _db->disconnect();
-    
+
     return true;
 }
 
@@ -413,16 +425,16 @@ HmSearchImpl::hash_string HmSearchImpl::get_multiple_keys(
                                                           int partition)
 {
     hash_string hashes;
-    
+
     pqxx::nontransaction n(*_db);
     pqxx::binarystring key_blob(key, _partition_bytes);
-    pqxx::result res = n.prepared("select_"+std::to_string(partition))(key_blob).exec();
-    
+    pqxx::result res = n.prepared("select_"+patch::to_string(static_cast<long long>(partition)))(key_blob).exec();
+
     for (pqxx::result::const_iterator c = res.begin(); c != res.end(); ++c) {
         pqxx::binarystring hash_result(c[0]);
         hashes.append(hash_string(hash_result.data(), hash_result.size()));
     }
-    
+
     return hashes;
 }
 
@@ -432,7 +444,7 @@ void HmSearchImpl::get_candidates(
 {
     uint8_t key[_partition_bytes];
     memset(key, 0, _partition_bytes);
-    
+
     for (int i = 0; i < _partitions; i++) {
         int psize = _hash_bits - i * _partition_bits;
         if (psize > _partition_bits) {
@@ -441,7 +453,7 @@ void HmSearchImpl::get_candidates(
         std::stringstream single;
         single << "SELECT hash FROM partition" << i
         << " WHERE key=$1";
-        
+
         std::stringstream s;
         s << "SELECT hash FROM partition"
         << i
@@ -456,28 +468,28 @@ void HmSearchImpl::get_candidates(
         << ".key = x.key";
         std::string sql;
         sql.append(s.str());
-        _db->prepare("select_multiple_"+std::to_string(i), sql);
-        
+        _db->prepare("select_multiple_"+patch::to_string(static_cast<long long>(i)), sql);
+
         sql.clear();
         sql.append(single.str());
-        _db->prepare("select_"+std::to_string(i), sql);
+        _db->prepare("select_"+patch::to_string(static_cast<long long>(i)), sql);
     }
     for (int i = 0; i < _partitions; i++) {
         hash_string hashes;
-        
+
         int bits = get_partition_key(query, i, key);
-        
+
         // Get exact matches
-        
+
         hashes = get_multiple_keys(key, i);
-        
+
         if (hashes.length() > 0) {
             add_hash_candidates(candidates, 0, (const uint8_t*)hashes.data(), hashes.length());
         }
-        
+
         // Get 1-variant matches
         pqxx::nontransaction n(*_db);
-        pqxx::prepare::invocation prep = n.prepared("select_multiple_"+std::to_string(i));
+        pqxx::prepare::invocation prep = n.prepared("select_multiple_"+patch::to_string(static_cast<long long>(i)));
         int pbyte = (i * _partition_bits) / 8;
         int count = 0;
         for (int pbit = i * _partition_bits; bits > 0; pbit++, bits--, count++) {
@@ -488,7 +500,7 @@ void HmSearchImpl::get_candidates(
             key[pbit / 8 - pbyte] ^= flip;
         }
         pqxx::result res = prep.exec();
-        
+
         hashes.clear();
         for (pqxx::result::const_iterator c = res.begin(); c != res.end(); ++c) {
             pqxx::binarystring hash_result(c[0]);
@@ -506,7 +518,7 @@ void HmSearchImpl::add_hash_candidates(
     for (size_t n = 0; n < length; n += _hash_bytes) {
         hash_string hash = hash_string(hashes + n, _hash_bytes);
         Candidate& cand = candidates[hash];
-        
+
         ++cand.matches;
         if (cand.matches == 1) {
             cand.first_match = match;
@@ -536,7 +548,7 @@ bool HmSearchImpl::valid_candidate(
                 return false;
         }
     }
-    
+
     return true;
 }
 
@@ -546,11 +558,11 @@ int HmSearchImpl::hamming_distance(
                                    const HmSearchImpl::hash_string& hash)
 {
     int distance = 0;
-    
+
     for (size_t i = 0; i < query.length(); i++) {
         distance += one_bits[query[i] ^ hash[i]];
     }
-    
+
     return distance;
 }
 
@@ -558,31 +570,31 @@ int HmSearchImpl::hamming_distance(
 int HmSearchImpl::get_partition_key(const hash_string& hash, int partition, uint8_t *key)
 {
     int psize, hash_bit, bits_left;
-    
+
     psize = _hash_bits - partition * _partition_bits;
     if (psize > _partition_bits) {
         psize = _partition_bits;
     }
-    
+
     // Copy bytes, masking out some bits at the start and end
     bits_left = psize;
     hash_bit = partition * _partition_bits;
-    
+
     for (int i = 0; i < _partition_bytes; i++) {
         int byte = hash_bit / 8;
         int bit = hash_bit % 8;
         int bits = 8 - bit;
-        
+
         if (bits > bits_left) {
             bits = bits_left;
         }
-        
+
         bits_left -= bits;
         hash_bit += bits;
-        
+
         key[i] = hash[byte] & (((1 << bits) - 1) << (8 - bit - bits));
     }
-    
+
     return psize;
 }
 

data/hmsearch-postgres.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'hmsearch-postgres'
-  s.version = '0.1.5'
+  s.version = '0.1.7'
 
   s.summary = 'hmsearch postgres client'
   s.description = <<-EOF

data/test/hmsearch_test.rb

@@ -4,37 +4,43 @@ require 'pg'
 
 class TestHmsearchPostgres < MiniTest::Unit::TestCase
   def setup
-    conn = PG::Connection.open('host=localhost port=5432')
+    conn = PG::Connection.open('host=localhost port=5432 user=postgres')
     conn.exec('drop database hmsearch_test') rescue nil
     conn.exec('create database hmsearch_test')
   end
 
   def test_hmsearch
-    HmSearch::Postgres.initdb('host=localhost port=5432 dbname=hmsearch_test', 256, 10, 100000000)
+    HmSearch::Postgres.initdb('host=localhost port=5432 user=postgres dbname=hmsearch_test', 256, 10, 100000000)
 
     conn = HmSearch::Postgres.open('host=localhost port=5432 dbname=hmsearch_test');
 
     conn.insert('6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df0')
-
-    # we can't do more than one op per open because of a bug in underlying library
-    # warning: adding 'int' to a string does not append to the string [-Wstring-plus-int]
-    # pqxx::prepare::invocation prep = n.prepared("select_multiple_"+i);
-    #
-    # instead of appending to the string this just moves the char pointer
-    # happens to work out ok one run through but not subsequent
+    conn.insert('6e6fb315fa8c43fe9c2687d5be14575a4d7252104236747d571b97e003563df0')
+
+    expected = [{
+      hash: '6e6fb315fa8c43fe9c2687d5be14575a4d7252104236747d571b97e003563df0',
+      distance: 7
+    },{
+      hash: '6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df0',
+      distance: 1
+    }]
+
+    actual = conn.lookup('6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df2') 
+    
     conn.close
 
-    HmSearch::Postgres.open('host=localhost port=5432 dbname=hmsearch_test') do |conn|
+    conn = nil
+
+
+    assert_equal(expected, actual)
 
-      actual = conn.lookup('6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df2', -1) 
-      
+    HmSearch::Postgres.open('host=localhost port=5432 dbname=hmsearch_test') do |conn|
       expected = [{
         hash: '6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df0',
         distance: 1
       }]
-
+      actual = conn.lookup('6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df2', 1)
       assert_equal(expected, actual)
-
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hmsearch-postgres
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Kris Selden
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-15 00:00:00.000000000 Z
+date: 2015-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake