hmsearch-postgres 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6819501caf23acccdaa5b8fd3f06087395c29322
+  data.tar.gz: 9d065fdd3196ba0404529f744d84a80f361c8124
+SHA512:
+  metadata.gz: bd3b9373805b316ddf0269fafe616cb22a2b70b40c39ad60c8d42783f2a6a08818ecd50ebf69a3e5f143d820f84272eb8322edd5c9771049f0509d3a8adcb51c
+  data.tar.gz: f1864f94b7069b97b646c84d4b32b04029951648dd64ee32497a1554178861d9ab687d9cc3a39fb12ad07d921539ea00e12e4b733c20bb676ad72684b4f81e1b

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+tmp
+lib/hmsearch/postgres_ext.*

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+ 
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Mine
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,27 @@
+require 'rake/clean'
+
+CLEAN.include('lib/hmsearch/Makefile')
+
+CLOBBER.include('lib/hmsearch/postgres_ext.*')
+
+task default: :test
+
+desc 'build native extension'
+task :build do
+  mkdir_p 'tmp'
+  chdir 'tmp' do
+    sh 'ruby ../ext/hmsearch/extconf.rb'
+    sh 'make'
+  end
+
+  require 'rbconfig'
+  dlext = RbConfig::CONFIG['DLEXT']
+  cp "tmp/postgres_ext.#{dlext}", "lib/hmsearch/"
+end
+
+desc 'run tests'
+task test: :build do
+  FileList.new('./test/*_test.rb').each do |test|
+    load test
+  end
+end

data/ext/hmsearch/extconf.rb

@@ -0,0 +1,6 @@
+require 'mkmf'
+require 'rbconfig'
+
+$LDFLAGS += ' -lm -lpqxx'
+
+create_makefile("hmsearch/postgres_ext")

data/ext/hmsearch/hmsearch.cc

@@ -0,0 +1,613 @@
+/* HmSearch hash lookup library
+ *
+ * http://commonsmachinery.se/
+ * Distributed under an MIT license
+ * 
+ * Copyright (c) 2014 Commons Machinery
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <math.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <memory>
+#include <algorithm>
+#include <map>
+#include <iostream>
+
+#include <pqxx/pqxx>
+
+#include "hmsearch.h"
+
+/** The actual implementation of the HmSearch database.
+ *
+ * A difference between this implementation and the HmSearch algorithm
+ * in the paper is that only exact-matches are stored in the database,
+ * not the 1-matches.  The 1-var partitions are instead generated
+ * during lookup.  This drastically reduces database size, which
+ * speeds up insertion and probably lookups too.
+ *
+ * The database contains some setting records controlling the
+ * operation:
+ *
+ * _hb: hash bits
+ * _me: max errors
+ * _se: global sequence when adding new keys
+ *
+ * These can't be changed once the database has been initialised.
+ *
+ * Each partition is stored as a key on the following format:
+ *  Byte 0: 'P'
+ *  Byte 1: Partition number (thus limiting to max error 518)
+ *  Bytes 2-N: Partition bits.
+ *  Bytes N-M: Sequence number (globally incremented with each addition)
+ */
+class HmSearchImpl : public HmSearch
+{
+public:
+    HmSearchImpl(std::string connstr, int hash_bits, int max_error)
+        : _hash_bits(hash_bits)
+        , _max_error(max_error)
+        , _hash_bytes((hash_bits + 7) / 8)
+        , _partitions((max_error + 3) / 2)
+        , _partition_bits(ceil((double)hash_bits / _partitions))
+        , _partition_bytes((_partition_bits + 7) / 8 + 1)
+    {
+      _db = new pqxx::connection(connstr);
+    }
+
+    ~HmSearchImpl() {
+        close();
+    }
+    
+    bool insert(const hash_string& hash,
+                std::string* error_msg = NULL);
+
+    bool print_copystring(const hash_string& hash,
+                std::string* error_msg = NULL);
+
+    bool lookup(const hash_string& query,
+                LookupResultList& result,
+                int max_error = -1,
+                std::string* error_msg = NULL);
+
+    bool close(std::string* error_msg = NULL);
+
+private:
+    struct Candidate {
+        Candidate() : matches(0), first_match(0), second_match(0) {}
+        int matches;
+        int first_match;
+        int second_match;
+    };
+
+    typedef std::map<hash_string, Candidate> CandidateMap;
+    
+    void get_candidates(const hash_string& query, CandidateMap& candidates);
+    void add_hash_candidates(CandidateMap& candidates, int match,
+                             const uint8_t* hashes, size_t length);
+    hash_string get_multiple_keys(uint8_t *key, int partition);
+    bool valid_candidate(const Candidate& candidate);
+    int hamming_distance(const hash_string& query, const hash_string& hash);
+    
+    int get_partition_key(const hash_string& hash, int partition, uint8_t *key);
+
+    pqxx::connection *_db;
+    int _hash_bits;
+    int _max_error;
+    int _hash_bytes;
+    int _partitions;
+    int _partition_bits;
+    int _partition_bytes;
+
+    static int one_bits[256];
+};
+
+
+bool HmSearch::init(const std::string& path,
+                    unsigned hash_bits, unsigned max_error,
+                    uint64_t num_hashes,
+                    std::string* error_msg)
+{
+    std::string dummy;
+    if (!error_msg) {
+        error_msg = &dummy;
+    }
+    *error_msg = "";
+
+    if (hash_bits == 0 || (hash_bits & 7)) {
+        *error_msg = "invalid hash_bits value";
+        return false;
+    }
+
+    if (max_error == 0 || max_error >= hash_bits || max_error > 518) {
+        *error_msg = "invalid max_error value";
+        return false;
+    }
+
+    pqxx::connection db(path);
+    if (!db.is_open()) {
+       *error_msg = "Can't open database";
+       return false;
+    }
+
+    std::string sql;
+    sql = "INSERT INTO config VALUES ($1, $2)";
+    db.prepare("hash_max", sql);
+
+    pqxx::work W(db);
+    sql = "CREATE TABLE IF NOT EXISTS config ("\
+          "       hash_bits int,"\
+          "       max_error int); TRUNCATE config";
+    W.exec(sql);
+
+    W.prepared("hash_max")(hash_bits)(max_error).exec();
+
+    for (unsigned int i = 0; i < ((max_error + 3) / 2); i++) {
+        {
+            std::stringstream s;
+            s << "CREATE TABLE IF NOT EXISTS partition" << i << " ("
+              << "       hash bytea,"
+              << "       key bytea); TRUNCATE partition" << i;
+            W.exec(s.str());
+        }
+
+        {
+            std::stringstream s;
+            s << "DROP INDEX IF EXISTS ix_key_" << i;
+            W.exec(s.str());
+        }
+
+        {
+            std::stringstream s;
+            s << "CREATE INDEX ix_key_" << i << " ON partition" << i << "(key)";
+            W.exec(s.str());
+        }
+    }
+
+    W.commit();
+
+    db.disconnect();
+
+    return true;
+}
+
+
+HmSearch* HmSearch::open(const std::string& path,
+                         std::string* error_msg)
+{
+    std::string dummy;
+
+    if (!error_msg) {
+        error_msg = &dummy;
+    }
+    *error_msg = "";
+
+    try {
+        pqxx::connection db(path);
+
+        std::string sql;
+
+        sql = "SELECT max_error, hash_bits FROM config";
+        pqxx::nontransaction n(db);
+        pqxx::result res(n.exec(sql));
+
+        pqxx::result::const_iterator c = res.begin(); // We retrieve just one row
+
+        unsigned long hash_bits, max_error;
+        max_error = c[0].as<long>();
+        hash_bits = c[1].as<long>();
+
+        db.disconnect();
+
+        HmSearch* hm = new HmSearchImpl(path, hash_bits, max_error);
+        if (!hm) {
+            *error_msg = "out of memory";
+            return NULL;
+        }
+
+        return hm;
+
+    }
+    catch (const pqxx::broken_connection& e) {
+        *error_msg = e.what();
+        return NULL;
+    }
+}
+
+
+HmSearch::hash_string HmSearch::parse_hexhash(const std::string& hexhash)
+{
+    int len = hexhash.length() / 2;
+    uint8_t hash[len];
+    
+    for (int i = 0; i < len; i++) {
+        char buf[3];
+        char* err;
+        
+        buf[0] = hexhash[i * 2];
+        buf[1] = hexhash[i * 2 + 1];
+        buf[2] = 0;
+
+        hash[i] = strtoul(buf, &err, 16);
+        
+        if (*err != '\0') {
+            return hash_string();
+        }
+    }
+
+    return hash_string(hash, len);
+}
+
+std::string HmSearch::format_hexhash(const HmSearch::hash_string& hash)
+{
+    char hex[hash.length() * 2 + 1];
+
+    for (size_t i = 0; i < hash.length(); i++) {
+        sprintf(hex + 2 * i, "%02x", hash[i]);
+    }
+
+    return hex;
+}
+
+
+bool HmSearchImpl::print_copystring(const hash_string& hash,
+                          std::string* error_msg)
+{
+    std::string dummy;
+    if (!error_msg) {
+        error_msg = &dummy;
+    }
+    *error_msg = "";
+
+    if (hash.length() != (size_t) _hash_bytes) {
+        *error_msg = "incorrect hash length";
+        return false;
+    }
+
+    for (int i = 0; i < _partitions; i++) {
+        uint8_t key[_partition_bytes];
+
+        get_partition_key(hash, i, key);
+
+        std::cout << "\\\\x" << format_hexhash(hash)
+                  << " "
+                  << int(i)
+                  << " "
+                  << "\\\\x" << format_hexhash(hash_string(key, _partition_bytes))
+                  << std::endl;
+
+    }
+
+    return true;
+}
+
+bool HmSearchImpl::insert(const hash_string& hash,
+                          std::string* error_msg)
+{
+    std::string dummy;
+    if (!error_msg) {
+        error_msg = &dummy;
+    }
+    *error_msg = "";
+
+    if (hash.length() != (size_t) _hash_bytes) {
+        *error_msg = "incorrect hash length";
+        return false;
+    }
+
+    if (!_db->is_open()) {
+        *error_msg = "database is closed";
+        return false;
+    }
+
+    for (int i = 0; i < _partitions; i++) {
+       std::stringstream s;
+       s << "INSERT INTO partition" << i
+         << "  VALUES ($1, $2)";
+       _db->prepare("insert_"+i, s.str());
+    }
+    pqxx::work W(*_db);
+    for (int i = 0; i < _partitions; i++) {
+        uint8_t key[_partition_bytes];
+
+        get_partition_key(hash, i, key);
+
+        pqxx::binarystring key_blob(key, _partition_bytes);
+        pqxx::binarystring hash_blob(hash.data(), hash.length());
+
+        W.prepared("insert_"+i)(hash_blob)(key_blob).exec();
+    }
+
+    W.commit();
+
+    return true;
+}
+
+
+bool HmSearchImpl::lookup(const hash_string& query,
+                          LookupResultList& result,
+                          int reduced_error,
+                          std::string* error_msg)
+{
+    std::string dummy;
+    if (!error_msg) {
+        error_msg = &dummy;
+    }
+    *error_msg = "";
+
+    if (query.length() != (size_t) _hash_bytes) {
+        *error_msg = "incorrect hash length";
+        return false;
+    }
+
+    if (!_db->is_open()) {
+        *error_msg = "database is closed";
+        return false;
+    }
+
+    try {
+        CandidateMap candidates;
+        get_candidates(query, candidates);
+
+        for (CandidateMap::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
+            if (valid_candidate(i->second)) {
+                int distance = hamming_distance(query, i->first);
+
+                if (distance <= _max_error
+                    && (reduced_error < 0 || distance <= reduced_error)) {
+                    result.push_back(LookupResult(i->first, distance));
+                }
+            }
+        }
+    }
+    catch (const pqxx::pqxx_exception &e) {
+        *error_msg = e.base().what();
+        return false;
+    }
+
+    return true;
+}
+
+
+bool HmSearchImpl::close(std::string* error_msg)
+{
+    std::string dummy;
+    if (!error_msg) {
+        error_msg = &dummy;
+    }
+    *error_msg = "";
+
+    if (!_db->is_open()) {
+        // Already closed
+        return true;
+    }
+
+    _db->disconnect();
+
+    return true;
+}
+
+
+HmSearchImpl::hash_string HmSearchImpl::get_multiple_keys(
+    uint8_t *key,
+    int partition)
+{
+    hash_string hashes;
+
+    pqxx::nontransaction n(*_db);
+    pqxx::binarystring key_blob(key, _partition_bytes);
+    pqxx::result res = n.prepared("select_"+partition)(key_blob).exec();
+
+    for (pqxx::result::const_iterator c = res.begin(); c != res.end(); ++c) {
+       pqxx::binarystring hash_result(c[0]);
+       hashes.append(hash_string(hash_result.data(), hash_result.size()));
+    }
+
+    return hashes;
+}
+
+void HmSearchImpl::get_candidates(
+    const HmSearchImpl::hash_string& query,
+    HmSearchImpl::CandidateMap& candidates)
+{
+    uint8_t key[_partition_bytes];
+
+    for (int i = 0; i < _partitions; i++) {
+        int psize = _hash_bits - i * _partition_bits;
+        if (psize > _partition_bits) {
+            psize = _partition_bits;
+        }
+        std::stringstream single;
+        single << "SELECT hash FROM partition" << i
+               << " WHERE key=$1";
+
+        std::stringstream s;
+        s << "SELECT hash FROM partition"
+          << i
+          << " INNER JOIN (SELECT $1::bytea AS key";
+        for (int j = 2; j <= psize; j++) {
+           s << " UNION ALL SELECT "
+             << "$"
+             << j;
+        }
+        s << ") AS x ON partition"
+          << i
+          << ".key = x.key";
+        std::string sql;
+        sql.append(s.str());
+        _db->prepare("select_multiple_"+i, sql);
+
+        sql.clear();
+        sql.append(single.str());
+        _db->prepare("select_"+i, sql);
+    }
+    for (int i = 0; i < _partitions; i++) {
+        hash_string hashes;
+        
+        int bits = get_partition_key(query, i, key);
+
+        // Get exact matches
+
+        hashes = get_multiple_keys(key, i);
+
+        if (hashes.length() > 0) {
+            add_hash_candidates(candidates, 0, (const uint8_t*)hashes.data(), hashes.length());
+        }
+
+        // Get 1-variant matches
+        pqxx::nontransaction n(*_db);
+        pqxx::prepare::invocation prep = n.prepared("select_multiple_"+i);
+        int pbyte = (i * _partition_bits) / 8;
+        int count = 0;
+        for (int pbit = i * _partition_bits; bits > 0; pbit++, bits--, count++) {
+            uint8_t flip = 1 << (7 - (pbit % 8));
+            key[pbit / 8 - pbyte] ^= flip;
+            pqxx::binarystring key_blob(key, _partition_bytes);
+            prep(key_blob);
+            key[pbit / 8 - pbyte] ^= flip;
+        }
+        pqxx::result res = prep.exec();
+
+        hashes.clear();
+        for (pqxx::result::const_iterator c = res.begin(); c != res.end(); ++c) {
+            pqxx::binarystring hash_result(c[0]);
+            hashes.append(hash_string(hash_result.data(), hash_result.size()));
+        }
+        add_hash_candidates(candidates, 1, (const uint8_t*)hashes.data(), hashes.length());
+    }
+}
+
+
+void HmSearchImpl::add_hash_candidates(
+    HmSearchImpl::CandidateMap& candidates, int match,
+    const uint8_t* hashes, size_t length)
+{
+    for (size_t n = 0; n < length; n += _hash_bytes) {
+        hash_string hash = hash_string(hashes + n, _hash_bytes);
+        Candidate& cand = candidates[hash];
+
+        ++cand.matches;
+        if (cand.matches == 1) {
+            cand.first_match = match;
+        }
+        else if (cand.matches == 2) {
+            cand.second_match = match;
+        }
+    }
+}
+
+
+bool HmSearchImpl::valid_candidate(
+    const HmSearchImpl::Candidate& candidate)
+{
+    if (_max_error & 1) {
+        // Odd k
+        if (candidate.matches < 3) {
+            if (candidate.matches == 1 || (candidate.first_match && candidate.second_match)) {
+                return false;
+            }
+        }
+    }
+    else {
+        // Even k
+        if (candidate.matches < 2) {
+            if (candidate.first_match)
+                return false;
+        }
+    }
+
+    return true;
+}
+
+
+int HmSearchImpl::hamming_distance(
+    const HmSearchImpl::hash_string& query,
+    const HmSearchImpl::hash_string& hash)
+{
+    int distance = 0;
+
+    for (size_t i = 0; i < query.length(); i++) {
+        distance += one_bits[query[i] ^ hash[i]];
+    }
+
+    return distance;
+}
+
+
+int HmSearchImpl::get_partition_key(const hash_string& hash, int partition, uint8_t *key)
+{
+    int psize, hash_bit, bits_left;
+
+    psize = _hash_bits - partition * _partition_bits;
+    if (psize > _partition_bits) {
+        psize = _partition_bits;
+    }
+
+    // Copy bytes, masking out some bits at the start and end
+    bits_left = psize;
+    hash_bit = partition * _partition_bits;
+
+    for (int i = 0; i < _partition_bytes; i++) {
+        int byte = hash_bit / 8;
+        int bit = hash_bit % 8;
+        int bits = 8 - bit;
+
+        if (bits > bits_left) {
+            bits = bits_left;
+        }
+
+        bits_left -= bits;
+        hash_bit += bits;
+
+        key[i] = hash[byte] & (((1 << bits) - 1) << (8 - bit - bits));
+    }
+
+    return psize;
+}
+
+
+int HmSearchImpl::one_bits[256] = {
+    0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+};
+
+/*
+  Local Variables:
+  c-file-style: "stroustrup"
+  indent-tabs-mode:nil
+  End:
+*/

data/ext/hmsearch/hmsearch.h

@@ -0,0 +1,204 @@
+/* HmSearch hash lookup library
+ *
+ * http://commonsmachinery.se/
+ * Distributed under an MIT license
+ * 
+ * Copyright (c) 2014 Commons Machinery
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __HMSEARCH_H_INCLUDED__
+#define __HMSEARCH_H_INCLUDED__
+
+#include <string>
+#include <list>
+#include <stdint.h>
+
+/** Interface to a HmSearch database.
+ *
+ * It cannot be instantiated directly, instead open() must be used to
+ * get a pointer to a database object.
+ *
+ * The database is closed when the object is deleted.  Until it is closed,
+ * any added hashes may not yet have been written to disk.
+ *
+ * Multiple threads can call HmSearch::insert() and HmSearch::lookup()
+ * on the same database object without locks.
+ *
+ * HmSearch::close() is not thread-safe, so the caller must ensure
+ * that no inserts or lookups are in progress.
+ *
+ * A database file can only be opened by a single process.  This is a
+ * limitation in the underlying LevelDB library.
+ */
+class HmSearch
+{
+public:
+    /** A string for passing around raw (i.e. not hexadecimal) hashes.
+     */
+    typedef std::basic_string<uint8_t> hash_string;
+
+    /** A record holding a hash found by lookup() and its hamming distance
+     * from the query hash.
+     */
+    struct LookupResult {
+        LookupResult(const hash_string& h, int d) : hash(h), distance(d) {}
+        hash_string hash;
+        int distance;
+    };
+
+    typedef std::list<LookupResult> LookupResultList;
+
+    /** Initialise a new hash database file.
+     *
+     * The database file should not exist, or if it does it must not
+     * contain any records.
+     *
+     * Parameters:
+     *
+     *  - path:       file path, typically ending in ".ldb"
+     *
+     *  - hash_bits:  number of bits in the hash (must be a multiple of 8)
+     *
+     *  - max_error:  maximum hamming distance, must be less than hash_bits
+     *
+     *  - num_hashes: target number of hashes, used for tuning the database
+     *
+     *  - error_msg:  if provided, will be set to an string describing any
+     *                error, or to an empty string if no error occurred.
+     *
+     * Returns true if the database could be initialised, false on errors.
+     */
+    static bool init(const std::string& path, 
+                     unsigned hash_bits, unsigned max_error,
+                     uint64_t num_hashes,
+                     std::string* error_msg = NULL);
+
+    /** Open a database file.
+     *
+     * The returned object must be deleted when not used any longer to
+     * ensure that the database is synced and closed.
+     * 
+     * Parameters:
+     *
+     *  - path: file path, typically ending in ".ldb"
+     *
+     *  - mode: database open mode
+     *
+     *  - error_msg:  if provided, will be set to an string describing any
+     *                error, or to an empty string if no error occurred.
+     *
+     * Returns the new object on success, or NULL on error.
+     */
+    static HmSearch* open(const std::string& path,
+                          std::string* error_msg = NULL);
+
+
+    /** Parse a hash in hexadecimal format, returning
+     * a string of raw bytes.
+     */
+    static hash_string parse_hexhash(const std::string& hexhash);
+
+    /** Format a hash of raw bytes into a hexadecimal string.
+     */
+    static std::string format_hexhash(const hash_string& hash);
+
+
+    /** Prints a string suitable for PostgreSQL COPY onto stdout.
+     *
+     * No check is made if the hash already exists in the database,
+     * so this may result in duplicate records.
+     *
+     * Parameters:
+     *  - hash:      The hash to print, as raw bytes
+     *  - error_msg: if provided, will be set to an string describing any
+     *               error, or to an empty string if no error occurred.
+     *
+     * Returns true if the command succeded, false on any error.
+     */
+    virtual bool print_copystring(const hash_string& hash,
+                        std::string* error_msg = NULL) = 0;
+
+    /** Insert a hash into the database.
+     *
+     * No check is made if the hash already exists in the database,
+     * so this may result in duplicate records.
+     *
+     * Parameters:
+     *  - hash:      The hash to insert, as raw bytes
+     *  - error_msg: if provided, will be set to an string describing any
+     *               error, or to an empty string if no error occurred.
+     *
+     * Returns true if the insert succeded, false on any error.
+     */
+    virtual bool insert(const hash_string& hash,
+                        std::string* error_msg = NULL) = 0;
+
+    /** Lookup a hash in the database, returning a list of matches.
+     *
+     * Parameters:
+     *
+     *  - query:     query hash string
+     *
+     *  - result:    matches are added to this list (which is not emptied)
+     *
+     *  - max_error: if >= 0, reduce the maximum accepted error
+     *               from the database default
+     *
+     *  - error_msg: if provided, will be set to an string describing any
+     *               error, or to an empty string if no error occurred.
+     *
+     * Returns true if the lookup could be performed (even if no
+     * hashes were found), false if an error occurred.
+     */
+    virtual bool lookup(const hash_string& query,
+                        LookupResultList& result,
+                        int max_error = -1,
+                        std::string* error_msg = NULL) = 0;
+
+    /** Explicitly sync and close the database file.
+     *
+     * Parameter:
+     *  - error_msg: if provided, will be set to an string describing any
+     *               error, or to an empty string if no error occurred.
+     *
+     * Returns true if all went well, false on errors.
+     */
+    virtual bool close(std::string* error_msg = NULL) = 0;
+
+    /** Delete the database object, syncing and closing the database
+     * file if not already done.
+     */
+    virtual ~HmSearch() {}
+
+protected:
+    HmSearch() {}
+};
+
+
+/*
+  Local Variables:
+  c-file-style: "stroustrup"
+  indent-tabs-mode:nil
+  End:
+*/
+
+#endif // __HMSEARCH_H_INCLUDED__
+

data/ext/hmsearch/postgres_ext.cc

@@ -0,0 +1,127 @@
+#include "ruby.h"
+#include "hmsearch.h"
+
+extern "C" void Init_postgres_ext();
+
+static VALUE eHmSearchError;
+static ID id_hash;
+static ID id_distance;
+
+static HmSearch *HmSearch_ptr(VALUE obj);
+static void HmSearch_free(void *p);
+static VALUE HmSearch_initdb(VALUE klass, VALUE _path, VALUE _hash_bits, VALUE _max_error, VALUE _num_hashes);
+static VALUE HmSearch_open(VALUE klass, VALUE _path);
+static VALUE HmSearch_close(VALUE self);
+static VALUE HmSearch_insert(VALUE self, VALUE _hash);
+static VALUE HmSearch_lookup(int argc, VALUE *argv, VALUE self);
+
+static HmSearch *HmSearch_ptr(VALUE obj)
+{
+  HmSearch *p;
+  Data_Get_Struct(obj, HmSearch, p);
+  return p;
+}
+
+static void HmSearch_free(void *p)
+{
+  static_cast<HmSearch*>(p)->~HmSearch();
+}
+
+static VALUE HmSearch_initdb(VALUE klass, VALUE _path, VALUE _hash_bits, VALUE _max_error, VALUE _num_hashes)
+{
+  const char *path = StringValueCStr(_path);
+  unsigned hash_bits = FIX2UINT(_hash_bits);
+  unsigned max_error = FIX2UINT(_max_error);
+  uint64_t num_hashes = NUM2ULONG(_num_hashes);
+
+  std::string error_msg;
+  if (!HmSearch::init(path, hash_bits, max_error, num_hashes, &error_msg)) {
+    rb_raise(eHmSearchError, "%s", error_msg.c_str());
+  }
+
+  return Qnil;
+}
+
+static VALUE HmSearch_open(VALUE klass, VALUE _path)
+{
+  const char *path = StringValueCStr(_path);
+
+  std::string error_msg;
+  HmSearch* p = HmSearch::open(path, &error_msg);
+  if (!p) {
+    rb_raise(eHmSearchError, "%s", error_msg.c_str());
+  }
+
+  VALUE obj = Data_Wrap_Struct(klass, NULL, HmSearch_free, p);
+
+  if (rb_block_given_p()) {
+    return rb_ensure(RUBY_METHOD_FUNC(rb_yield), obj, RUBY_METHOD_FUNC(HmSearch_close), obj);
+  }
+
+  return obj;
+}
+
+static VALUE HmSearch_close(VALUE self)
+{
+  HmSearch_ptr(self)->close();
+  return Qnil;
+}
+
+static VALUE HmSearch_insert(VALUE self, VALUE _hash)
+{
+  HmSearch_ptr(self)->insert(HmSearch::parse_hexhash(StringValueCStr(_hash)));
+  return Qnil;
+}
+
+static VALUE HmSearch_lookup(int argc, VALUE *argv, VALUE self)
+{
+  VALUE _hash, _reduced_error;
+  rb_scan_args(argc, argv, "11", &_hash, &_reduced_error);
+
+  const char *hash = StringValueCStr(_hash);
+
+  int reduced_error = NIL_P(_reduced_error) ? -1 : FIX2INT(_reduced_error);
+
+  HmSearch::LookupResultList matches;
+  std::string error_msg;
+  if (!HmSearch_ptr(self)->lookup(HmSearch::parse_hexhash(hash), matches, reduced_error, &error_msg)) {
+    rb_raise(eHmSearchError, "%s", error_msg.c_str());
+  }
+
+  VALUE results = rb_ary_new2(matches.size());
+
+  for (HmSearch::LookupResultList::const_iterator it=matches.begin(); it != matches.end(); ++it) {
+    VALUE result = rb_hash_new();
+
+    rb_hash_aset(result, ID2SYM(id_hash), rb_str_new2(HmSearch::format_hexhash(it->hash).c_str()));
+    rb_hash_aset(result, ID2SYM(id_distance), INT2NUM(it->distance));
+
+    rb_ary_push(results, result);
+  }
+
+  return results;
+}
+
+void Init_postgres_ext() {
+  VALUE mHmSearch = rb_define_module("HmSearch");
+  
+  eHmSearchError = rb_define_class_under(mHmSearch, "HmSearchError", rb_eStandardError);
+
+  VALUE cPostgres = rb_define_class_under(mHmSearch, "Postgres", rb_cObject);
+
+  VALUE cPostgresSingleton = rb_singleton_class(cPostgres);
+
+  id_hash = rb_intern("hash");
+  id_distance = rb_intern("distance");
+
+  rb_define_method(cPostgresSingleton, "initdb", RUBY_METHOD_FUNC(HmSearch_initdb), 4);
+
+  rb_define_method(cPostgresSingleton, "open",  RUBY_METHOD_FUNC(HmSearch_open), 1);
+  // we have to allocate via open
+  rb_undef_method(cPostgresSingleton, "new");
+  rb_undef_method(cPostgresSingleton, "allocate");
+
+  rb_define_method(cPostgres, "close", RUBY_METHOD_FUNC(HmSearch_close), 0);
+  rb_define_method(cPostgres, "insert", RUBY_METHOD_FUNC(HmSearch_insert), 1);
+  rb_define_method(cPostgres, "lookup", RUBY_METHOD_FUNC(HmSearch_lookup), -1);
+}

data/hmsearch-postgres.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name = 'hmsearch-postgres'
+  s.version = '0.1.0'
+
+  s.summary = 'hmsearch postgres client'
+  s.description = <<-EOF
+    A ruby wrapper for the hmsearch postgres client.
+  EOF
+
+  s.files = `git ls-files`.split("\n")
+  s.require_path = 'lib'
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'pg'
+
+  s.extensions = 'ext/hmsearch/extconf.rb'
+
+  s.authors = ['Kris Selden']
+  s.email = 'kris.selden@gmail.com'
+  s.homepage = 'https://github.com/denisnazarov/hmsearch-postgres-ruby'
+  s.license = 'MIT'
+end

data/lib/hmsearch/postgres.rb

@@ -0,0 +1 @@
+require 'hmsearch/postgres_ext'

data/test/hmsearch_test.rb

@@ -0,0 +1,40 @@
+require 'hmsearch/postgres'
+require 'minitest/autorun'
+require 'pg'
+
+class TestHmsearchPostgres < MiniTest::Unit::TestCase
+  def setup
+    conn = PG::Connection.open('host=localhost port=5432')
+    conn.exec('drop database hmsearch_test') rescue nil
+    conn.exec('create database hmsearch_test')
+  end
+
+  def test_hmsearch
+    HmSearch::Postgres.initdb('host=localhost port=5432 dbname=hmsearch_test', 256, 10, 100000000)
+
+    conn = HmSearch::Postgres.open('host=localhost port=5432 dbname=hmsearch_test');
+
+    conn.insert('6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df0')
+
+    # we can't do more than one op per open because of a bug in underlying library
+    # warning: adding 'int' to a string does not append to the string [-Wstring-plus-int]
+    # pqxx::prepare::invocation prep = n.prepared("select_multiple_"+i);
+    #
+    # instead of appending to the string this just moves the char pointer
+    # happens to work out ok one run through but not subsequent
+    conn.close
+
+    HmSearch::Postgres.open('host=localhost port=5432 dbname=hmsearch_test') do |conn|
+
+      actual = conn.lookup('6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df2', -1) 
+      
+      expected = [{
+        hash: '6e6fb315fa8c43fe9c2687d5be14575abb7252104236747d571b97e003563df0',
+        distance: 1
+      }]
+
+      assert_equal(expected, actual)
+
+    end
+  end
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: hmsearch-postgres
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kris Selden
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+      A ruby wrapper for the hmsearch postgres client.
+email: kris.selden@gmail.com
+executables: []
+extensions:
+- ext/hmsearch/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- ext/hmsearch/extconf.rb
+- ext/hmsearch/hmsearch.cc
+- ext/hmsearch/hmsearch.h
+- ext/hmsearch/postgres_ext.cc
+- hmsearch-postgres.gemspec
+- lib/hmsearch/postgres.rb
+- test/hmsearch_test.rb
+homepage: https://github.com/denisnazarov/hmsearch-postgres-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: hmsearch postgres client
+test_files: []