command-t 1.7 → 1.8

data/ruby/command-t/depend

@@ -1,4 +1,4 @@
-# Copyright 2010 Wincent Colaiuta. All rights reserved.
+# Copyright 2010-2014 Wincent Colaiuta. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -21,4 +21,4 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-CFLAGS += -std=c99 -Wall -Wextra -Wno-unused-parameter
+CFLAGS += -Wall -Wextra -Wno-unused-parameter

data/ruby/command-t/ext.c

@@ -1,4 +1,4 @@
-// Copyright 2010-2013 Wincent Colaiuta. All rights reserved.
+// Copyright 2010-2014 Wincent Colaiuta. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
@@ -22,15 +22,19 @@
 // POSSIBILITY OF SUCH DAMAGE.
 
 #include "matcher.h"
+#include "watchman.h"
 
-VALUE mCommandT         = 0; // module CommandT
-VALUE cCommandTMatcher  = 0; // class CommandT::Matcher
+VALUE mCommandT              = 0; // module CommandT
+VALUE cCommandTMatcher       = 0; // class CommandT::Matcher
+VALUE mCommandTWatchman      = 0; // module CommandT::Watchman
+VALUE mCommandTWatchmanUtils = 0; // module CommandT::Watchman::Utils
 
 VALUE CommandT_option_from_hash(const char *option, VALUE hash)
 {
+    VALUE key;
     if (NIL_P(hash))
         return Qnil;
-    VALUE key = ID2SYM(rb_intern(option));
+    key = ID2SYM(rb_intern(option));
     if (rb_funcall(hash, rb_intern("has_key?"), 1, key) == Qtrue)
         return rb_hash_aref(hash, key);
     else
@@ -44,8 +48,13 @@ void Init_ext()
 
     // class CommandT::Matcher
     cCommandTMatcher = rb_define_class_under(mCommandT, "Matcher", rb_cObject);
-
-    // methods
     rb_define_method(cCommandTMatcher, "initialize", CommandTMatcher_initialize, -1);
     rb_define_method(cCommandTMatcher, "sorted_matches_for", CommandTMatcher_sorted_matches_for, -1);
+
+    // module CommandT::Watchman::Utils
+    mCommandTWatchman = rb_define_module_under(mCommandT, "Watchman");
+    mCommandTWatchmanUtils = rb_define_module_under(mCommandTWatchman, "Utils");
+    rb_define_singleton_method(mCommandTWatchmanUtils, "load", CommandTWatchmanUtils_load, 1);
+    rb_define_singleton_method(mCommandTWatchmanUtils, "dump", CommandTWatchmanUtils_dump, 1);
+    rb_define_singleton_method(mCommandTWatchmanUtils, "query", CommandTWatchmanUtils_query, 2);
 }

data/ruby/command-t/ext.h

@@ -1,4 +1,4 @@
-// Copyright 2010-2013 Wincent Colaiuta. All rights reserved.
+// Copyright 2010-2014 Wincent Colaiuta. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
@@ -23,8 +23,10 @@
 
 #include <ruby.h>
 
-extern VALUE mCommandT;         // module CommandT
-extern VALUE cCommandTMatcher;  // class CommandT::Matcher
+extern VALUE mCommandT;              // module CommandT
+extern VALUE cCommandTMatcher;       // class CommandT::Matcher
+extern VALUE mCommandTWatchman;      // module CommandT::Watchman
+extern VALUE mCommandTWatchmanUtils; // module CommandT::Watchman::Utils
 
 // Encapsulates common pattern of checking for an option in an optional
 // options hash. The hash itself may be nil, but an exception will be

data/ruby/command-t/extconf.rb

@@ -36,6 +36,16 @@ header('ruby.h')
 header('stdlib.h')
 header('string.h')
 
+# optional headers (for CommandT::Watchman::Utils)
+if have_header('fcntl.h') &&
+  have_header('sys/errno.h') &&
+  have_header('sys/socket.h')
+  RbConfig::MAKEFILE_CONFIG['DEFS'] += ' -DWATCHMAN_BUILD'
+
+  have_header('ruby/st.h') # >= 1.9; sets HAVE_RUBY_ST_H
+  have_header('st.h')      # 1.8; sets HAVE_ST_H
+end
+
 # optional
 if RbConfig::CONFIG['THREAD_MODEL'] == 'pthread'
   have_library('pthread', 'pthread_create') # sets HAVE_PTHREAD_H if found

data/ruby/command-t/match.c

@@ -48,6 +48,9 @@ double recursive_match(matchinfo_t *m,    // sharable meta-data
     double seen_score = 0;  // remember best score seen via recursion
     int dot_file_match = 0; // true if needle matches a dot-file
     int dot_search = 0;     // true if searching for a dot
+    long i, j, distance;
+    int found;
+    double score_for_char;
 
     // do we have a memoized result we can return?
     double memoized = m->memo[needle_idx * m->needle_len + haystack_idx];
@@ -60,15 +63,15 @@ double recursive_match(matchinfo_t *m,    // sharable meta-data
         goto memoize;
     }
 
-    for (long i = needle_idx; i < m->needle_len; i++) {
+    for (i = needle_idx; i < m->needle_len; i++) {
         char c = m->needle_p[i];
         if (c == '.')
             dot_search = 1;
-        int found = 0;
+        found = 0;
 
         // similar to above, we'll stop iterating when we know we're too close
         // to the end of the string to possibly match
-        for (long j = haystack_idx;
+        for (j = haystack_idx;
              j <= m->haystack_len - (m->needle_len - i);
              j++, haystack_idx++) {
             char d = m->haystack_p[j];
@@ -87,8 +90,8 @@ double recursive_match(matchinfo_t *m,    // sharable meta-data
                 dot_search = 0;
 
                 // calculate score
-                double score_for_char = m->max_score_per_char;
-                long distance = j - last_idx;
+                score_for_char = m->max_score_per_char;
+                distance = j - last_idx;
 
                 if (distance > 1) {
                     double factor = 1.0;
@@ -152,6 +155,8 @@ void calculate_match(VALUE str,
                      VALUE never_show_dot_files,
                      match_t *out)
 {
+    long i, max;
+    double score;
     matchinfo_t m;
     m.haystack_p            = RSTRING_PTR(str);
     m.haystack_len          = RSTRING_LEN(str);
@@ -163,14 +168,14 @@ void calculate_match(VALUE str,
     m.never_show_dot_files  = never_show_dot_files == Qtrue;
 
     // calculate score
-    double score = 1.0;
+    score = 1.0;
 
     // special case for zero-length search string
     if (m.needle_len == 0) {
 
         // filter out dot files
         if (!m.always_show_dot_files) {
-            for (long i = 0; i < m.haystack_len; i++) {
+            for (i = 0; i < m.haystack_len; i++) {
                 char c = m.haystack_p[i];
 
                 if (c == '.' && (i == 0 || m.haystack_p[i - 1] == '/')) {
@@ -183,7 +188,7 @@ void calculate_match(VALUE str,
 
         // prepare for memoization
         double memo[m.haystack_len * m.needle_len];
-        for (long i = 0, max = m.haystack_len * m.needle_len; i < max; i++)
+        for (i = 0, max = m.haystack_len * m.needle_len; i < max; i++)
             memo[i] = DBL_MAX;
         m.memo = memo;
 

data/ruby/command-t/matcher.c

@@ -1,4 +1,4 @@
-// Copyright 2010-2013 Wincent Colaiuta. All rights reserved.
+// Copyright 2010-2014 Wincent Colaiuta. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
@@ -77,9 +77,12 @@ int cmp_score(const void *a, const void *b)
 
 VALUE CommandTMatcher_initialize(int argc, VALUE *argv, VALUE self)
 {
-    // process arguments: 1 mandatory, 1 optional
-    VALUE scanner, options;
+    VALUE always_show_dot_files;
+    VALUE never_show_dot_files;
+    VALUE options;
+    VALUE scanner;
 
+    // process arguments: 1 mandatory, 1 optional
     if (rb_scan_args(argc, argv, "11", &scanner, &options) == 1)
         options = Qnil;
     if (NIL_P(scanner))
@@ -88,8 +91,8 @@ VALUE CommandTMatcher_initialize(int argc, VALUE *argv, VALUE self)
     rb_iv_set(self, "@scanner", scanner);
 
     // check optional options hash for overrides
-    VALUE always_show_dot_files = CommandT_option_from_hash("always_show_dot_files", options);
-    VALUE never_show_dot_files = CommandT_option_from_hash("never_show_dot_files", options);
+    always_show_dot_files = CommandT_option_from_hash("always_show_dot_files", options);
+    never_show_dot_files = CommandT_option_from_hash("never_show_dot_files", options);
 
     rb_iv_set(self, "@always_show_dot_files", always_show_dot_files);
     rb_iv_set(self, "@never_show_dot_files", never_show_dot_files);
@@ -110,8 +113,9 @@ typedef struct {
 
 void *match_thread(void *thread_args)
 {
+    long i;
     thread_args_t *args = (thread_args_t *)thread_args;
-    for (long i = args->thread_index; i < args->path_count; i += args->thread_count) {
+    for (i = args->thread_index; i < args->path_count; i += args->thread_count) {
         VALUE path = RARRAY_PTR(args->paths)[i];
         calculate_match(path,
                         args->abbrev,
@@ -126,9 +130,23 @@ void *match_thread(void *thread_args)
 
 VALUE CommandTMatcher_sorted_matches_for(int argc, VALUE *argv, VALUE self)
 {
-    // process arguments: 1 mandatory, 1 optional
-    VALUE abbrev, options;
+    long i, limit, path_count, thread_count;
+#ifdef HAVE_PTHREAD_H
+    long err;
+#endif
+    match_t *matches;
+    thread_args_t *thread_args;
+    VALUE abbrev;
+    VALUE always_show_dot_files;
+    VALUE limit_option;
+    VALUE never_show_dot_files;
+    VALUE options;
+    VALUE paths;
+    VALUE results;
+    VALUE scanner;
+    VALUE threads_option;
 
+    // process arguments: 1 mandatory, 1 optional
     if (rb_scan_args(argc, argv, "11", &abbrev, &options) == 1)
         options = Qnil;
     if (NIL_P(abbrev))
@@ -138,22 +156,21 @@ VALUE CommandTMatcher_sorted_matches_for(int argc, VALUE *argv, VALUE self)
     abbrev = rb_funcall(abbrev, rb_intern("downcase"), 0);
 
     // check optional options has for overrides
-    VALUE limit_option = CommandT_option_from_hash("limit", options);
-    VALUE threads_option = CommandT_option_from_hash("threads", options);
+    limit_option = CommandT_option_from_hash("limit", options);
+    threads_option = CommandT_option_from_hash("threads", options);
 
     // get unsorted matches
-    VALUE scanner = rb_iv_get(self, "@scanner");
-    VALUE paths = rb_funcall(scanner, rb_intern("paths"), 0);
-    VALUE always_show_dot_files = rb_iv_get(self, "@always_show_dot_files");
-    VALUE never_show_dot_files = rb_iv_get(self, "@never_show_dot_files");
+    scanner = rb_iv_get(self, "@scanner");
+    paths = rb_funcall(scanner, rb_intern("paths"), 0);
+    always_show_dot_files = rb_iv_get(self, "@always_show_dot_files");
+    never_show_dot_files = rb_iv_get(self, "@never_show_dot_files");
 
-    long path_count = RARRAY_LEN(paths);
-    match_t *matches = malloc(path_count * sizeof(match_t));
+    path_count = RARRAY_LEN(paths);
+    matches = malloc(path_count * sizeof(match_t));
     if (!matches)
         rb_raise(rb_eNoMemError, "memory allocation failed");
 
-    int err;
-    long thread_count = NIL_P(threads_option) ? 1 : NUM2LONG(threads_option);
+    thread_count = NIL_P(threads_option) ? 1 : NUM2LONG(threads_option);
 
 #ifdef HAVE_PTHREAD_H
 #define THREAD_THRESHOLD 1000 /* avoid the overhead of threading when search space is small */
@@ -164,10 +181,10 @@ VALUE CommandTMatcher_sorted_matches_for(int argc, VALUE *argv, VALUE self)
         rb_raise(rb_eNoMemError, "memory allocation failed");
 #endif
 
-    thread_args_t *thread_args = malloc(sizeof(thread_args_t) * thread_count);
+    thread_args = malloc(sizeof(thread_args_t) * thread_count);
     if (!thread_args)
         rb_raise(rb_eNoMemError, "memory allocation failed");
-    for (int i = 0; i < thread_count; i++) {
+    for (i = 0; i < thread_count; i++) {
         thread_args[i].thread_count = thread_count;
         thread_args[i].thread_index = i;
         thread_args[i].matches = matches;
@@ -186,16 +203,16 @@ VALUE CommandTMatcher_sorted_matches_for(int argc, VALUE *argv, VALUE self)
         } else {
             err = pthread_create(&threads[i], NULL, match_thread, (void *)&thread_args[i]);
             if (err != 0)
-                rb_raise(rb_eSystemCallError, "pthread_create() failure (%d)", err);
+                rb_raise(rb_eSystemCallError, "pthread_create() failure (%d)", (int)err);
         }
 #endif
     }
 
 #ifdef HAVE_PTHREAD_H
-    for (int i = 0; i < thread_count - 1; i++) {
+    for (i = 0; i < thread_count - 1; i++) {
         err = pthread_join(threads[i], NULL);
         if (err != 0)
-            rb_raise(rb_eSystemCallError, "pthread_join() failure (%d)", err);
+            rb_raise(rb_eSystemCallError, "pthread_join() failure (%d)", (int)err);
     }
     free(threads);
 #endif
@@ -208,12 +225,12 @@ VALUE CommandTMatcher_sorted_matches_for(int argc, VALUE *argv, VALUE self)
         // for all other non-empty search strings, sort by score
         qsort(matches, path_count, sizeof(match_t), cmp_score);
 
-    VALUE results = rb_ary_new();
+    results = rb_ary_new();
 
-    long limit = NIL_P(limit_option) ? 0 : NUM2LONG(limit_option);
+    limit = NIL_P(limit_option) ? 0 : NUM2LONG(limit_option);
     if (limit == 0)
         limit = path_count;
-    for (long i = 0; i < path_count && limit > 0; i++) {
+    for (i = 0; i < path_count && limit > 0; i++) {
         if (matches[i].score > 0.0) {
             rb_funcall(results, rb_intern("push"), 1, matches[i].path);
             limit--;

data/ruby/command-t/scanner/file_scanner/watchman_file_scanner.rb

@@ -21,7 +21,7 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-require 'json'
+require 'pathname'
 require 'socket'
 require 'command-t/vim'
 require 'command-t/vim/path_utilities'
@@ -43,24 +43,27 @@ module CommandT
       def paths
         @paths[@path] ||= begin
           ensure_cache_under_limit
-          sockname = JSON[%x{watchman get-sockname}]['sockname']
+          sockname = Watchman::Utils.load(
+            %x{watchman --output-encoding=bser get-sockname}
+          )['sockname']
           raise WatchmanUnavailable unless $?.exitstatus.zero?
-          socket = UNIXSocket.open(sockname) do |s|
-            root = File.realpath(File.expand_path(@path))
-            s.puts JSON.generate(['watch-list'])
-            if !JSON[s.gets]['roots'].include?(root)
+
+          UNIXSocket.open(sockname) do |socket|
+            root = Pathname.new(@path).realpath.to_s
+            roots = Watchman::Utils.query(['watch-list'], socket)['roots']
+            if !roots.include?(root)
               # this path isn't being watched yet; try to set up watch
-              s.puts JSON.generate(['watch', root])
+              result = Watchman::Utils.query(['watch', root], socket)
 
               # root_restrict_files setting may prevent Watchman from working
-              raise WatchmanUnavailable if JSON[s.gets].has_key?('error')
+              raise WatchmanUnavailable if result.has_key?('error')
             end
 
-            s.puts JSON.generate(['query', root, {
+            query = ['query', root, {
               'expression' => ['type', 'f'],
               'fields'     => ['name'],
-            }])
-            paths = JSON[s.gets]
+            }]
+            paths = Watchman::Utils.query(query, socket)
 
             # could return error if watch is removed
             raise WatchmanUnavailable if paths.has_key?('error')

data/ruby/command-t/watchman.c

@@ -0,0 +1,660 @@
+// Copyright 2014 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "watchman.h"
+
+#ifdef WATCHMAN_BUILD
+
+#if defined(HAVE_RUBY_ST_H)
+#include <ruby/st.h>
+#elif defined(HAVE_ST_H)
+#include <st.h>
+#else
+#error no st.h header found
+#endif
+
+#include <fcntl.h>      /* for fcntl() */
+#include <sys/errno.h>  /* for errno */
+#include <sys/socket.h> /* for recv(), MSG_PEEK */
+
+typedef struct {
+    uint8_t *data;  // payload
+    size_t cap;     // total capacity
+    size_t len;     // current length
+} watchman_t;
+
+// Forward declarations:
+VALUE watchman_load(char **ptr, char *end);
+void watchman_dump(watchman_t *w, VALUE serializable);
+
+/**
+ * Inspect a ruby object (debugging aid)
+ */
+#define ruby_inspect(obj) rb_funcall(rb_mKernel, rb_intern("p"), 1, obj)
+
+/**
+ * Print `count` bytes of memory at `address` (debugging aid)
+ */
+#define dump(address, count) \
+    do { \
+        for (int i = 0; i < count; i++) { \
+            printf("%02x ", ((unsigned char *)address)[i]); printf("\n"); \
+        } \
+    } while(0)
+
+#define WATCHMAN_DEFAULT_STORAGE 4096
+
+#define WATCHMAN_BINARY_MARKER   "\x00\x01"
+#define WATCHMAN_ARRAY_MARKER    0x00
+#define WATCHMAN_HASH_MARKER     0x01
+#define WATCHMAN_STRING_MARKER   0x02
+#define WATCHMAN_INT8_MARKER     0x03
+#define WATCHMAN_INT16_MARKER    0x04
+#define WATCHMAN_INT32_MARKER    0x05
+#define WATCHMAN_INT64_MARKER    0x06
+#define WATCHMAN_FLOAT_MARKER    0x07
+#define WATCHMAN_TRUE            0x08
+#define WATCHMAN_FALSE           0x09
+#define WATCHMAN_NIL             0x0a
+#define WATCHMAN_TEMPLATE_MARKER 0x0b
+#define WATCHMAN_SKIP_MARKER     0x0c
+
+#define WATCHMAN_HEADER \
+        WATCHMAN_BINARY_MARKER \
+        "\x06" \
+        "\x00\x00\x00\x00\x00\x00\x00\x00"
+
+static const char watchman_array_marker  = WATCHMAN_ARRAY_MARKER;
+static const char watchman_hash_marker   = WATCHMAN_HASH_MARKER;
+static const char watchman_string_marker = WATCHMAN_STRING_MARKER;
+static const char watchman_true          = WATCHMAN_TRUE;
+static const char watchman_false         = WATCHMAN_FALSE;
+static const char watchman_nil           = WATCHMAN_NIL;
+
+/**
+ * Appends `len` bytes, starting at `data`, to the watchman_t struct `w`
+ *
+ * Will attempt to reallocate the underlying storage if it is not sufficient.
+ */
+void watchman_append(watchman_t *w, const char *data, size_t len) {
+    if (w->len + len > w->cap) {
+        w->cap += w->len + WATCHMAN_DEFAULT_STORAGE;
+        REALLOC_N(w->data, uint8_t, w->cap);
+    }
+    memcpy(w->data + w->len, data, len);
+    w->len += len;
+}
+
+/**
+ * Allocate a new watchman_t struct
+ *
+ * The struct has a small amount of extra capacity preallocated, and a blank
+ * header that can be filled in later to describe the PDU.
+ */
+watchman_t *watchman_init() {
+    watchman_t *w = ALLOC(watchman_t);
+    w->cap = WATCHMAN_DEFAULT_STORAGE;
+    w->len = 0;
+    w->data = ALLOC_N(uint8_t, WATCHMAN_DEFAULT_STORAGE);
+
+    watchman_append(w, WATCHMAN_HEADER, sizeof(WATCHMAN_HEADER) - 1);
+    return w;
+}
+
+/**
+ * Free a watchman_t struct `w` that was previously allocated with
+ * `watchman_init`
+ */
+void watchman_free(watchman_t *w) {
+    xfree(w->data);
+    xfree(w);
+}
+
+/**
+ * Encodes and appends the integer `num` to `w`
+ */
+void watchman_dump_int(watchman_t *w, int64_t num) {
+    char encoded[1 + sizeof(int64_t)];
+
+    if (num == (int8_t)num) {
+        encoded[0] = WATCHMAN_INT8_MARKER;
+        encoded[1] = (int8_t)num;
+        watchman_append(w, encoded, 1 + sizeof(int8_t));
+    } else if (num == (int16_t)num) {
+        encoded[0] = WATCHMAN_INT16_MARKER;
+        *(int16_t *)(encoded + 1) = (int16_t)num;
+        watchman_append(w, encoded, 1 + sizeof(int16_t));
+    } else if (num == (int32_t)num) {
+        encoded[0] = WATCHMAN_INT32_MARKER;
+        *(int32_t *)(encoded + 1) = (int32_t)num;
+        watchman_append(w, encoded, 1 + sizeof(int32_t));
+    } else {
+        encoded[0] = WATCHMAN_INT64_MARKER;
+        *(int64_t *)(encoded + 1) = (int64_t)num;
+        watchman_append(w, encoded, 1 + sizeof(int64_t));
+    }
+}
+
+/**
+ * Encodes and appends the string `string` to `w`
+ */
+void watchman_dump_string(watchman_t *w, VALUE string) {
+    watchman_append(w, &watchman_string_marker, sizeof(watchman_string_marker));
+    watchman_dump_int(w, RSTRING_LEN(string));
+    watchman_append(w, RSTRING_PTR(string), RSTRING_LEN(string));
+}
+
+/**
+ * Encodes and appends the double `num` to `w`
+ */
+void watchman_dump_double(watchman_t *w, double num) {
+    char encoded[1 + sizeof(double)];
+    encoded[0] = WATCHMAN_FLOAT_MARKER;
+    *(double *)(encoded + 1) = num;
+    watchman_append(w, encoded, sizeof(encoded));
+}
+
+/**
+ * Encodes and appends the array `array` to `w`
+ */
+void watchman_dump_array(watchman_t *w, VALUE array) {
+    long i;
+    watchman_append(w, &watchman_array_marker, sizeof(watchman_array_marker));
+    watchman_dump_int(w, RARRAY_LEN(array));
+    for (i = 0; i < RARRAY_LEN(array); i++) {
+        watchman_dump(w, rb_ary_entry(array, i));
+    }
+}
+
+/**
+ * Helper method that encodes and appends a key/value pair (`key`, `value`) from
+ * a hash to the watchman_t struct passed in via `data`
+ */
+int watchman_dump_hash_iterator(VALUE key, VALUE value, VALUE data) {
+    watchman_t *w = (watchman_t *)data;
+    watchman_dump_string(w, StringValue(key));
+    watchman_dump(w, value);
+    return ST_CONTINUE;
+}
+
+/**
+ * Encodes and appends the hash `hash` to `w`
+ */
+void watchman_dump_hash(watchman_t *w, VALUE hash) {
+    watchman_append(w, &watchman_hash_marker, sizeof(watchman_hash_marker));
+    watchman_dump_int(w, RHASH_SIZE(hash));
+    rb_hash_foreach(hash, watchman_dump_hash_iterator, (VALUE)w);
+}
+
+/**
+ * Encodes and appends the serialized Ruby object `serializable` to `w`
+ *
+ * Examples of serializable objects include arrays, hashes, strings, numbers
+ * (integers, floats), booleans, and nil.
+ */
+void watchman_dump(watchman_t *w, VALUE serializable) {
+    switch (TYPE(serializable)) {
+        case T_ARRAY:
+            return watchman_dump_array(w, serializable);
+        case T_HASH:
+            return watchman_dump_hash(w, serializable);
+        case T_STRING:
+            return watchman_dump_string(w, serializable);
+        case T_FIXNUM: // up to 63 bits
+            return watchman_dump_int(w, FIX2LONG(serializable));
+        case T_BIGNUM:
+            return watchman_dump_int(w, NUM2LL(serializable));
+        case T_FLOAT:
+            return watchman_dump_double(w, NUM2DBL(serializable));
+        case T_TRUE:
+            return watchman_append(w, &watchman_true, sizeof(watchman_true));
+        case T_FALSE:
+            return watchman_append(w, &watchman_false, sizeof(watchman_false));
+        case T_NIL:
+            return watchman_append(w, &watchman_nil, sizeof(watchman_nil));
+        default:
+            rb_raise(rb_eTypeError, "unsupported type");
+    }
+}
+
+/**
+ * Extract and return the int encoded at `ptr`
+ *
+ * Moves `ptr` past the extracted int.
+ *
+ * Will raise an ArgumentError if extracting the int would take us beyond the
+ * end of the buffer indicated by `end`, or if there is no int encoded at `ptr`.
+ *
+ * @returns The extracted int
+ */
+int64_t watchman_load_int(char **ptr, char *end) {
+    char *val_ptr = *ptr + sizeof(int8_t);
+    int64_t val = 0;
+
+    if (val_ptr >= end) {
+        rb_raise(rb_eArgError, "insufficient int storage");
+    }
+
+    switch (*ptr[0]) {
+        case WATCHMAN_INT8_MARKER:
+            if (val_ptr + sizeof(int8_t) > end) {
+                rb_raise(rb_eArgError, "overrun extracting int8_t");
+            }
+            val = *(int8_t *)val_ptr;
+            *ptr = val_ptr + sizeof(int8_t);
+            break;
+        case WATCHMAN_INT16_MARKER:
+            if (val_ptr + sizeof(int16_t) > end) {
+                rb_raise(rb_eArgError, "overrun extracting int16_t");
+            }
+            val = *(int16_t *)val_ptr;
+            *ptr = val_ptr + sizeof(int16_t);
+            break;
+        case WATCHMAN_INT32_MARKER:
+            if (val_ptr + sizeof(int32_t) > end) {
+                rb_raise(rb_eArgError, "overrun extracting int32_t");
+            }
+            val = *(int32_t *)val_ptr;
+            *ptr = val_ptr + sizeof(int32_t);
+            break;
+        case WATCHMAN_INT64_MARKER:
+            if (val_ptr + sizeof(int64_t) > end) {
+                rb_raise(rb_eArgError, "overrun extracting int64_t");
+            }
+            val = *(int64_t *)val_ptr;
+            *ptr = val_ptr + sizeof(int64_t);
+            break;
+        default:
+            rb_raise(rb_eArgError, "bad integer marker 0x%02x", (unsigned int)*ptr[0]);
+            break;
+    }
+
+    return val;
+}
+
+/**
+ * Reads and returns a string encoded in the Watchman binary protocol format,
+ * starting at `ptr` and finishing at or before `end`
+ */
+VALUE watchman_load_string(char **ptr, char *end) {
+    if (*ptr >= end) {
+        rb_raise(rb_eArgError, "unexpected end of input");
+    }
+
+    if (*ptr[0] != WATCHMAN_STRING_MARKER) {
+        rb_raise(rb_eArgError, "not a number");
+    }
+
+    *ptr += sizeof(int8_t);
+    if (*ptr >= end) {
+        rb_raise(rb_eArgError, "invalid string header");
+    }
+
+    int64_t len = watchman_load_int(ptr, end);
+    if (len == 0) { // special case for zero-length strings
+        return rb_str_new2("");
+    } else if (*ptr + len > end) {
+        rb_raise(rb_eArgError, "insufficient string storage");
+    }
+
+    VALUE string = rb_str_new(*ptr, len);
+    *ptr += len;
+    return string;
+}
+
+/**
+ * Reads and returns a double encoded in the Watchman binary protocol format,
+ * starting at `ptr` and finishing at or before `end`
+ */
+double watchman_load_double(char **ptr, char *end) {
+    *ptr += sizeof(int8_t); // caller has already verified the marker
+    if (*ptr + sizeof(double) > end) {
+        rb_raise(rb_eArgError, "insufficient double storage");
+    }
+    double val = *(double *)*ptr;
+    *ptr += sizeof(double);
+    return val;
+}
+
+/**
+ * Helper method which returns length of the array encoded in the Watchman
+ * binary protocol format, starting at `ptr` and finishing at or before `end`
+ */
+int64_t watchman_load_array_header(char **ptr, char *end) {
+    if (*ptr >= end) {
+        rb_raise(rb_eArgError, "unexpected end of input");
+    }
+
+    // verify and consume marker
+    if (*ptr[0] != WATCHMAN_ARRAY_MARKER) {
+        rb_raise(rb_eArgError, "not an array");
+    }
+    *ptr += sizeof(int8_t);
+
+    // expect a count
+    if (*ptr + sizeof(int8_t) * 2 > end) {
+        rb_raise(rb_eArgError, "incomplete array header");
+    }
+    return watchman_load_int(ptr, end);
+}
+
+/**
+ * Reads and returns an array encoded in the Watchman binary protocol format,
+ * starting at `ptr` and finishing at or before `end`
+ */
+VALUE watchman_load_array(char **ptr, char *end) {
+    int64_t count, i;
+    VALUE array;
+
+    count = watchman_load_array_header(ptr, end);
+    array = rb_ary_new2(count);
+
+    for (i = 0; i < count; i++) {
+        rb_ary_push(array, watchman_load(ptr, end));
+    }
+
+    return array;
+}
+
+/**
+ * Reads and returns a hash encoded in the Watchman binary protocol format,
+ * starting at `ptr` and finishing at or before `end`
+ */
+VALUE watchman_load_hash(char **ptr, char *end) {
+    int64_t count, i;
+    VALUE hash, key, value;
+
+    *ptr += sizeof(int8_t); // caller has already verified the marker
+
+    // expect a count
+    if (*ptr + sizeof(int8_t) * 2 > end) {
+        rb_raise(rb_eArgError, "incomplete hash header");
+    }
+    count = watchman_load_int(ptr, end);
+
+    hash = rb_hash_new();
+
+    for (i = 0; i < count; i++) {
+        key = watchman_load_string(ptr, end);
+        value = watchman_load(ptr, end);
+        rb_hash_aset(hash, key, value);
+    }
+
+    return hash;
+}
+
+/**
+ * Reads and returns a templated array encoded in the Watchman binary protocol
+ * format, starting at `ptr` and finishing at or before `end`
+ *
+ * Templated arrays are arrays of hashes which have repetitive key information
+ * pulled out into a separate "headers" prefix.
+ *
+ * @see https://github.com/facebook/watchman/blob/master/BSER.markdown
+ */
+VALUE watchman_load_template(char **ptr, char *end) {
+    int64_t header_items_count, i, row_count;
+    VALUE array, hash, header, key, value;
+
+    *ptr += sizeof(int8_t); // caller has already verified the marker
+
+    // process template header array
+    header_items_count = watchman_load_array_header(ptr, end);
+    header = rb_ary_new2(header_items_count);
+    for (i = 0; i < header_items_count; i++) {
+        rb_ary_push(header, watchman_load_string(ptr, end));
+    }
+
+    // process row items
+    row_count = watchman_load_int(ptr, end);
+    array = rb_ary_new2(header_items_count);
+    while (row_count--) {
+        hash = rb_hash_new();
+        for (i = 0; i < header_items_count; i++) {
+            if (*ptr >= end) {
+                rb_raise(rb_eArgError, "unexpected end of input");
+            }
+
+            if (*ptr[0] == WATCHMAN_SKIP_MARKER) {
+                *ptr += sizeof(uint8_t);
+            } else {
+                value = watchman_load(ptr, end);
+                key = rb_ary_entry(header, i);
+                rb_hash_aset(hash, key, value);
+            }
+        }
+        rb_ary_push(array, hash);
+    }
+    return array;
+}
+
+/**
+ * Reads and returns an object encoded in the Watchman binary protocol format,
+ * starting at `ptr` and finishing at or before `end`
+ */
+VALUE watchman_load(char **ptr, char *end) {
+    if (*ptr >= end) {
+        rb_raise(rb_eArgError, "unexpected end of input");
+    }
+
+    switch (*ptr[0]) {
+        case WATCHMAN_ARRAY_MARKER:
+            return watchman_load_array(ptr, end);
+        case WATCHMAN_HASH_MARKER:
+            return watchman_load_hash(ptr, end);
+        case WATCHMAN_STRING_MARKER:
+            return watchman_load_string(ptr, end);
+        case WATCHMAN_INT8_MARKER:
+        case WATCHMAN_INT16_MARKER:
+        case WATCHMAN_INT32_MARKER:
+        case WATCHMAN_INT64_MARKER:
+            return LL2NUM(watchman_load_int(ptr, end));
+        case WATCHMAN_FLOAT_MARKER:
+            return rb_float_new(watchman_load_double(ptr, end));
+        case WATCHMAN_TRUE:
+            *ptr += 1;
+            return Qtrue;
+        case WATCHMAN_FALSE:
+            *ptr += 1;
+            return Qfalse;
+        case WATCHMAN_NIL:
+            *ptr += 1;
+            return Qnil;
+        case WATCHMAN_TEMPLATE_MARKER:
+            return watchman_load_template(ptr, end);
+        default:
+            rb_raise(rb_eTypeError, "unsupported type");
+    }
+
+    return Qnil; // keep the compiler happy
+}
+
+/**
+ * CommandT::Watchman::Utils.load(serialized)
+ *
+ * Converts the binary object, `serialized`, from the Watchman binary protocol
+ * format into a normal Ruby object.
+ */
+VALUE CommandTWatchmanUtils_load(VALUE self, VALUE serialized) {
+    serialized = StringValue(serialized);
+    long len = RSTRING_LEN(serialized);
+    char *ptr = RSTRING_PTR(serialized);
+    char *end = ptr + len;
+
+    // expect at least the binary marker and a int8_t length counter
+    if ((size_t)len < sizeof(WATCHMAN_BINARY_MARKER) - 1 + sizeof(int8_t) * 2) {
+        rb_raise(rb_eArgError, "undersized header");
+    }
+
+    if (memcmp(ptr, WATCHMAN_BINARY_MARKER, sizeof(WATCHMAN_BINARY_MARKER) - 1)) {
+        rb_raise(rb_eArgError, "missing binary marker");
+    }
+
+    // get size marker
+    ptr += sizeof(WATCHMAN_BINARY_MARKER) - 1;
+    uint64_t payload_size = watchman_load_int(&ptr, end);
+    if (!payload_size) {
+        rb_raise(rb_eArgError, "empty payload");
+    }
+
+    // sanity check length
+    if (ptr + payload_size != end) {
+        rb_raise(rb_eArgError, "payload size mismatch (%lu)", end - (ptr + payload_size));
+    }
+
+    VALUE loaded = watchman_load(&ptr, end);
+
+    // one more sanity check
+    if (ptr != end) {
+        rb_raise(rb_eArgError, "payload termination mismatch (%lu)", end - ptr);
+    }
+
+    return loaded;
+}
+
+/**
+ * CommandT::Watchman::Utils.dump(serializable)
+ *
+ * Converts the Ruby object, `serializable`, into a binary string in the
+ * Watchman binary protocol format.
+ *
+ * Examples of serializable objects include arrays, hashes, strings, numbers
+ * (integers, floats), booleans, and nil.
+ */
+VALUE CommandTWatchmanUtils_dump(VALUE self, VALUE serializable) {
+    watchman_t *w = watchman_init();
+    watchman_dump(w, serializable);
+
+    // update header with final length information
+    uint64_t *len = (uint64_t *)(w->data + sizeof(WATCHMAN_HEADER) - sizeof(uint64_t) - 1);
+    *len = w->len - sizeof(WATCHMAN_HEADER) + 1;
+
+    // prepare final return value
+    VALUE serialized = rb_str_buf_new(w->len);
+    rb_str_buf_cat(serialized, (const char*)w->data, w->len);
+    watchman_free(w);
+    return serialized;
+}
+
+/**
+ * Helper method for raising a SystemCallError wrapping a lower-level error code
+ * coming from the `errno` global variable.
+ */
+void watchman_raise_system_call_error(int number) {
+    VALUE error = INT2FIX(number);
+    rb_exc_raise(rb_class_new_instance(1, &error, rb_eSystemCallError));
+}
+
+// How far we have to look to figure out the size of the PDU header
+#define WATCHMAN_SNIFF_BUFFER_SIZE sizeof(WATCHMAN_BINARY_MARKER) - 1 + sizeof(int8_t)
+
+// How far we have to peek, at most, to figure out the size of the PDU itself
+#define WATCHMAN_PEEK_BUFFER_SIZE \
+    sizeof(WATCHMAN_BINARY_MARKER) - 1 + \
+    sizeof(WATCHMAN_INT64_MARKER) + \
+    sizeof(int64_t)
+
+/**
+ * CommandT::Watchman::Utils.query(query, socket)
+ *
+ * Converts `query`, a Watchman query comprising Ruby objects, into the Watchman
+ * binary protocol format, transmits it over socket, and unserializes and
+ * returns the result.
+ */
+VALUE CommandTWatchmanUtils_query(VALUE self, VALUE query, VALUE socket) {
+    int fileno = NUM2INT(rb_funcall(socket, rb_intern("fileno"), 0));
+
+    // do blocking I/O to simplify the following logic
+    int flags = fcntl(fileno, F_GETFL);
+    if (fcntl(fileno, F_SETFL, flags & ~O_NONBLOCK) == -1) {
+        rb_raise(rb_eRuntimeError, "unable to clear O_NONBLOCK flag");
+    }
+
+    // send the message
+    VALUE serialized = CommandTWatchmanUtils_dump(self, query);
+    long query_len = RSTRING_LEN(serialized);
+    ssize_t sent = send(fileno, RSTRING_PTR(serialized), query_len, 0);
+    if (sent == -1) {
+        watchman_raise_system_call_error(errno);
+    } else if (sent != query_len) {
+        rb_raise(rb_eRuntimeError, "expected to send %ld bytes but sent %ld",
+            query_len, sent);
+    }
+
+    // sniff to see how large the header is
+    int8_t peek[WATCHMAN_PEEK_BUFFER_SIZE];
+    ssize_t received = recv(fileno, peek, WATCHMAN_SNIFF_BUFFER_SIZE, MSG_PEEK | MSG_WAITALL);
+    if (received == -1) {
+        watchman_raise_system_call_error(errno);
+    } else if (received != WATCHMAN_SNIFF_BUFFER_SIZE) {
+        rb_raise(rb_eRuntimeError, "failed to sniff PDU header");
+    }
+
+    // peek at size of PDU
+    int8_t sizes[] = { 0, 0, 0, 1, 2, 4, 8 };
+    ssize_t peek_size = sizeof(WATCHMAN_BINARY_MARKER) - 1 + sizeof(int8_t) +
+        sizes[peek[sizeof(WATCHMAN_BINARY_MARKER) - 1]];
+
+    received = recv(fileno, peek, peek_size, MSG_PEEK);
+    if (received == -1) {
+        watchman_raise_system_call_error(errno);
+    } else if (received != peek_size) {
+        rb_raise(rb_eRuntimeError, "failed to peek at PDU header");
+    }
+    int8_t *pdu_size_ptr = peek + sizeof(WATCHMAN_BINARY_MARKER) - sizeof(int8_t);
+    int64_t payload_size =
+        peek_size +
+        watchman_load_int((char **)&pdu_size_ptr, (char *)peek + peek_size);
+
+    // actually read the PDU
+    void *buffer = xmalloc(payload_size);
+    if (!buffer) {
+        rb_raise(rb_eNoMemError, "failed to allocate %lld bytes", payload_size);
+    }
+    received = recv(fileno, buffer, payload_size, MSG_WAITALL);
+    if (received == -1) {
+        watchman_raise_system_call_error(errno);
+    } else if (received != payload_size) {
+        rb_raise(rb_eRuntimeError, "failed to load PDU");
+    }
+    char *payload = buffer + peek_size;
+    VALUE loaded = watchman_load(&payload, payload + payload_size);
+    free(buffer);
+    return loaded;
+}
+
+#else /* don't build Watchman utils; supply stubs only*/
+
+VALUE CommandTWatchmanUtils_load(VALUE self, VALUE serialized) {
+    rb_raise(rb_eRuntimeError, "unsupported operation");
+}
+
+VALUE CommandTWatchmanUtils_dump(VALUE self, VALUE serializable) {
+    rb_raise(rb_eRuntimeError, "unsupported operation");
+}
+
+VALUE CommandTWatchmanUtils_query(VALUE self, VALUE query, VALUE socket) {
+    rb_raise(rb_eRuntimeError, "unsupported operation");
+}
+
+#endif