entangler 0.1.0

data/lib/entangler.rb

@@ -0,0 +1,23 @@
+require 'entangler/version'
+require 'entangler/entangled_file'
+
+module Entangler
+  class << self
+    attr_accessor :executor
+
+    def run(base_dir, opts = {})
+      opts = {mode: 'master'}.merge(opts)
+
+      require 'entangler/executor/base'
+      if opts[:mode] == 'master'
+        require 'entangler/executor/master'
+        self.executor = Entangler::Executor::Master.new(base_dir, opts)
+      elsif opts[:mode] == 'slave'
+        require 'entangler/executor/slave'
+        self.executor = Entangler::Executor::Slave.new(base_dir, opts)
+      end
+
+      self.executor.run
+    end
+  end
+end

data/lib/notifier/src/darwin/BUILD

@@ -0,0 +1,7 @@
+To compile notify.c, run
+
+clang -framework CoreFoundation -framework CoreServices -o notify notify.c
+
+or (if you don't have clang)
+
+gcc -framework CoreFoundation -framework CoreServices -o notify notify.c

data/lib/notifier/src/darwin/README

@@ -0,0 +1,12 @@
+The notify utility continiously prints changes in a specified directory in the following format:
+
+M changed/path/1
+M changed/path/2
+-
+
+
+Where M stands for "Modifed", and "-" is the indicator of end of changeset
+
+Utility only returns specific directories that have changed, it does not print
+the files/directories that have been modified: you need to determine them
+yourself

data/lib/notifier/src/darwin/notify.c

@@ -0,0 +1,63 @@
+#include <CoreServices/CoreServices.h>
+#include <CoreFoundation/CoreFoundation.h>
+#include <sys/stat.h>
+
+static void printChangesFunc(ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents, void *eventPaths, const FSEventStreamEventFlags eventFlags[], const FSEventStreamEventId eventIds[]) {
+	char **paths = eventPaths;
+	int i;
+	for (i = 0; i < numEvents; i++) {
+		printf("M %s\n", paths[i]);
+	}
+	printf("-\n");
+	fflush(stdout);
+}
+
+void initFSEvents(const char *path) {
+
+	/* Define variables and create a CFArray object containing
+	 CFString objects containing paths to watch.
+	 */
+
+	CFStringRef mypath = CFStringCreateWithCString(NULL, path, kCFStringEncodingUTF8);
+	CFArrayRef pathsToWatch = CFArrayCreate(NULL, (const void **)&mypath, 1, NULL);
+	void *callbackInfo = NULL; // could put stream-specific data here.
+	FSEventStreamRef stream;
+	CFAbsoluteTime latency = 0.1; /* Latency in seconds */
+
+	/* Create the stream, passing in a callback */
+	stream = FSEventStreamCreate(NULL,
+	                             &printChangesFunc,
+	                             callbackInfo,
+	                             pathsToWatch,
+	                             kFSEventStreamEventIdSinceNow, /* Or a previous event ID */
+	                             latency,
+	                             kFSEventStreamCreateFlagNone /* Flags explained in reference */
+	                             );
+
+	CFRelease(pathsToWatch);
+	CFRelease(mypath);
+
+	/* Create the stream before calling this. */
+	FSEventStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode);
+	FSEventStreamStart(stream);
+}
+
+int main (int argc, const char * argv[]) {
+
+	if(argc != 2) {
+		printf("Usage: %s <path>\n", argv[0]);
+		return 1;
+	}
+
+	struct stat tmp;
+
+	if(stat(argv[1], &tmp) != 0) {
+		perror("Invalid path");
+		return 2;
+	}
+
+	initFSEvents(argv[1]);
+	CFRunLoopRun();
+
+	return 0;
+}

data/lib/notifier/src/linux/BUILD

@@ -0,0 +1,4 @@
+notify.c was built using dietlibc to be as small as possible (using "bin-i386/diet gcc -static -o notify notify.c").
+If you want to compile it without dietlibc, you can just do
+
+  gcc -o notify notify.c

data/lib/notifier/src/linux/README

@@ -0,0 +1,18 @@
+The notify utility continiously prints changes in a specified directory in the following format:
+
+M changed/path/1
+M changed/path/2
+-
+
+Where M stands for "Modifed", and "-" is the indicator of end of changeset
+
+Utility only returns specific directories that have changed, it does not print
+the files/directories that have been modified: you need to determine them
+yourself.
+
+Linux-specific notes:
+
+    1. You might need to adjust '/proc/sys/fs/inotify/max_user_watches' to allow more directories to be watched
+    2. If you have a lot of changes you might also want to increase queue size in /proc/sys/fs/inotify/max_queued_events
+    3. Watched queue can overflow and notify utility can run out of watched directories.
+       When you get exit code 3, you need to restart the daemon and re-process all directories you are interested in.

data/lib/notifier/src/linux/notify.c

@@ -0,0 +1,296 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+
+/* gcc should be able to optimize strlen() for constant strings */
+#define PRINT(str) write(1, str, strlen(str))
+#define ERROR(str) write(2, str, strlen(str))
+
+#define EVENT_MASK (IN_CLOSE_WRITE|IN_CREATE|IN_DELETE|IN_DELETE_SELF|IN_MODIFY|IN_MOVE_SELF\
+    |IN_MOVED_FROM|IN_MOVED_TO|IN_DONT_FOLLOW|IN_ONLYDIR|IN_ATTRIB)
+
+#ifdef DEBUG
+#define DEBUG_PRINT printf
+#else
+#define DEBUG_PRINT(...) /* printf actually adds about 7k to binary size :) */
+#endif
+
+typedef struct {
+    int wd;
+    int parent_wd;
+    char *name;
+} _watchstruct;
+
+char events_buf[PATH_MAX + sizeof(struct inotify_event) + 1];
+static _watchstruct *watches;
+int ifd = 0, max_watches;
+char *watch_dir;
+
+/* make directory path for watch descriptor (recursively) */
+static void wd_path(int wd, char *path)
+{
+    if (wd == 0) {
+        strcpy(path, watch_dir);
+        strcat(path, "/");
+        return;
+    }
+
+    if (wd < 0 || !watches[wd].name) {
+        DEBUG_PRINT("Recusive %d, %x\n", wd, watches[wd].name);
+        ERROR("Memory corrupted: asked path of deleted event\n");
+        exit(1);
+    }
+
+    wd_path(watches[wd].parent_wd, path);
+    if (watches[wd].name[0] == 0) return;
+
+    strcat(path, watches[wd].name);
+    strcat(path, "/");
+}
+
+static int add_dir_watch(int parent_wd, char *dir, char *dir_name, int no_print)
+{
+    int wd = inotify_add_watch(ifd, dir, EVENT_MASK);
+    if (wd < 0) {
+        ERROR("Cannot add watch to '");
+        ERROR(dir);
+        ERROR("' using inotify: ");
+        if (errno == ENOSPC) {
+            ERROR("too many watches\nYou can increase number of user watches using /proc/sys/fs/inotify/max_user_watches");
+        } else {
+            ERROR(strerror(errno));
+        }
+        ERROR("\n");
+        if (errno != EACCES && errno != ENOENT) exit(1);
+        return wd;
+    }
+
+    if (wd >= max_watches) {
+        ERROR("\nToo many events; restart required to prevent watch descriptor overflow.\n");
+        exit(3);
+    }
+
+    dir_name = strdup(dir_name);
+    if (!dir_name) {
+        ERROR("Cannot strdup(dir_name)\n");
+        exit(1);
+    }
+
+    watches[wd].wd = wd;
+    watches[wd].parent_wd = parent_wd;
+    if (watches[wd].name) free(watches[wd].name);
+    watches[wd].name = dir_name;
+
+    if (!no_print) {
+        PRINT("M ");
+        PRINT(dir);
+        PRINT("\n");
+    }
+
+    return wd;
+}
+
+static void add_dir(int dir_wd, char *dir, int errors_fatal, int no_print)
+{
+    char path[PATH_MAX + 1];
+    DIR *dh = opendir(dir);
+    struct dirent *ent;
+    struct stat st;
+    int dirl = strlen(dir), n = sizeof(path) - 1 - dirl, had_errors = 0, wd;
+
+    if (dirl > sizeof(path) - 3) {
+        ERROR("Too long path (not watched): ");
+        ERROR(dir);
+        ERROR("\n");
+
+        if (errors_fatal) exit(1);
+        return;
+    }
+
+    if (!dh) {
+        ERROR("Cannot opendir(");
+        ERROR(dir);
+        ERROR("): ");
+        ERROR(strerror(errno));
+        ERROR("\n");
+
+        if (errors_fatal) exit(1);
+        return;
+    }
+
+    strcpy(path, dir);
+
+    while ((ent = readdir(dh)) != NULL) {
+        if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..") || !strcmp(ent->d_name, ".unrealsync")) continue;
+
+        path[dirl] = '/';
+        path[dirl + 1] = 0;
+        strncat(path + dirl, ent->d_name, n);
+        path[sizeof(path) - 1] = 0;
+        if (lstat(path, &st)) {
+            ERROR("Cannot lstat(");
+            ERROR(path);
+            ERROR("): ");
+            ERROR(strerror(errno));
+            ERROR("\n");
+            had_errors = 1;
+            continue;
+        }
+
+        if (S_ISDIR(st.st_mode)) {
+            wd = add_dir_watch(dir_wd, path, ent->d_name, no_print);
+            if (wd < 0) continue;
+            add_dir(wd, path, errors_fatal, no_print);
+        }
+    }
+
+    closedir(dh);
+
+    if (errors_fatal && had_errors) exit(1);
+}
+
+void debug_print_mask(uint32_t mask)
+{
+    if (mask & IN_DELETE_SELF) DEBUG_PRINT("IN_DELETE_SELF ");
+    if (mask & IN_MOVE_SELF)   DEBUG_PRINT("IN_MOVE_SELF ");
+    if (mask & IN_MOVED_FROM)  DEBUG_PRINT("IN_MOVED_FROM ");
+    if (mask & IN_MOVED_TO)    DEBUG_PRINT("IN_MOVED_TO ");
+    if (mask & IN_CLOSE_WRITE) DEBUG_PRINT("IN_CLOSE_WRITE ");
+    if (mask & IN_MODIFY)      DEBUG_PRINT("IN_MODIFY ");
+    if (mask & IN_IGNORED)     DEBUG_PRINT("IN_IGNORED ");
+    if (mask & IN_ISDIR)       DEBUG_PRINT("IN_ISDIR ");
+    if (mask & IN_Q_OVERFLOW)  DEBUG_PRINT("IN_Q_OVERFLOW ");
+    if (mask & IN_UNMOUNT)     DEBUG_PRINT("IN_UNMOUNT ");
+    if (mask & IN_CREATE)      DEBUG_PRINT("IN_CREATE ");
+}
+
+static int do_watch(int max_watches)
+{
+    struct inotify_event *ev = (struct inotify_event*)events_buf;
+    ssize_t n = 0, wd;
+    char path[PATH_MAX + 1];
+
+    watches = (_watchstruct*) calloc(max_watches, sizeof(_watchstruct));
+    if (!watches) {
+        ERROR("Cannot allocate memory\n");
+        exit(1);
+    }
+
+    DEBUG_PRINT("Doing initial watches setup\n");
+
+    ifd = inotify_init();
+    if (ifd == -1) {
+        perror("Cannot init inotify");
+        exit(1);
+    }
+
+    wd = add_dir_watch(0, watch_dir, "", 1);
+    if (wd < 0) {
+        ERROR("Cannot add dir watch\n");
+        exit(1);
+    }
+    add_dir(wd, watch_dir, 1, 1);
+
+    while ((n = read(ifd, events_buf, sizeof(events_buf))) > 0) {
+        ev = (struct inotify_event*)events_buf;
+        while (n > 0) {
+            if (ev->mask & IN_Q_OVERFLOW) {
+                ERROR("Queue overflow, restart needed\n");
+                exit(3);
+            }
+
+            if (ev->mask & IN_IGNORED) {
+                free(watches[ev->wd].name);
+                watches[ev->wd].parent_wd = -1;
+                watches[ev->wd].name = NULL;
+                goto loop_end;
+            }
+
+            wd_path(ev->wd, path);
+            PRINT("M ");
+            PRINT(path);
+            PRINT("\n");
+            #ifdef DEBUG
+            if (ev->len) {
+                DEBUG_PRINT(" | ");
+                DEBUG_PRINT("%s", ev->name);
+            }
+            DEBUG_PRINT(" | ");
+            debug_print_mask(ev->mask);
+            #endif
+
+            if ((ev->mask & IN_DELETE) || (ev->mask & IN_MOVED_FROM)) {
+                goto loop_end;
+            }
+
+            if (ev->mask & IN_ISDIR) {
+                if (ev->len + strlen(path) > sizeof(path) - 1) {
+                    ERROR("Too deep directory: ");
+                    ERROR(path);
+                    ERROR(ev->name);
+                    ERROR("\n");
+                    goto loop_end;
+                }
+                strcat(path, ev->name);
+                wd = add_dir_watch(ev->wd, path, ev->name, 0);
+                if (wd < 0) goto loop_end;
+                add_dir(ev->wd, path, 0, 0);
+            }
+
+            loop_end:
+            n -= sizeof(struct inotify_event) + ev->len;
+            ev = (struct inotify_event*) ((char*)ev + sizeof(struct inotify_event) + ev->len);
+        }
+
+        PRINT("-\n");
+    }
+
+    perror("Cannot read() inotify queue");
+    exit(1);
+}
+
+int main(int argc, char *argv[])
+{
+    int fd, n;
+    char buf[12];
+
+    if (argc != 2) {
+        ERROR("Usage: notify <dir>\n");
+        return 1;
+    }
+
+    fd = open("/proc/sys/fs/inotify/max_user_watches", O_RDONLY);
+    if (fd < 0) {
+        perror("Cannot open /proc/sys/fs/inotify/max_user_watches");
+        return 1;
+    }
+
+    if ( (n = read(fd, buf, sizeof(buf) - 1)) < 0) {
+        perror("Cannot read() /proc/sys/fs/inotify/max_user_watches");
+        return 1;
+    }
+
+    buf[n] = 0;
+    max_watches = atoi(buf) * 2;
+    if (max_watches <= 0) {
+        ERROR("Incorrect number of watches: ");
+        ERROR(buf);
+        ERROR("\n");
+        return 1;
+    } else {
+        DEBUG_PRINT("Max watches: %d\n", max_watches);
+    }
+
+    watch_dir = argv[1];
+    do_watch(max_watches);
+
+    return 0;
+}

data/lib/notifier/src/linux/old/BUILD

@@ -0,0 +1,11 @@
+The utility has been built using the following command under Ubuntu Linux 11.10 32-bit:
+
+  g++ -static -s -I. *.cpp /usr/lib/libpopt.a -o notify
+
+Or, to achieve the minimal size of the binary:
+
+  g++ -static -s -Os -ffunction-sections -fdata-sections -Wl,--gc-sections -I. *.cpp /usr/lib/libpopt.a -o notify
+
+You might need to change path to libpopt to whatever location it is on your system.
+Libpopt is the only dependency aside from g++ if you want to build the utility.
+It comes pre-built statically with realsync, so it will most probably just work

data/lib/notifier/src/linux/old/README

@@ -0,0 +1,14 @@
+This is a daemon that notifies about filesystem changes in the following form:
+
+M /path/to/changed/file
+M /path/to/other/changed/file
+-
+
+The daemon is based on kfsmd-0.3.3 and tuned a little bit to be more realtime
+The original kfsmd-0.3.3 could be found at http://sourceforge.net/projects/witme/files/kfsmd/
+The preferred way to use this daemon is to run the following:
+
+./notify watch /path/to/watch | uniq
+
+You can get rid of "| uniq", but in this case you will get multiple lines per each file
+for each type of event (e.g. OPEN/WRITE/CLOSE)

data/lib/notifier/src/linux/old/kernel-filesystem-monitor-daemon-cat.cpp

@@ -0,0 +1,165 @@
+/******************************************************************************
+*******************************************************************************
+*******************************************************************************
+
+
+    kernel-filesystem-monitor-daemon-cat
+    Copyright (C) 2005 Ben Martin
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+    For more details see the COPYING file in the root directory of this
+    distribution.
+
+    $Id: kernel-filesystem-monitor-daemon-cat.cpp,v 1.3 2008/05/25 21:30:52 ben Exp $
+
+*******************************************************************************
+*******************************************************************************
+******************************************************************************/
+
+#include <iostream>
+#include "kernel-filesystem-monitor-daemon.hh"
+#include <stdlib.h>
+
+using namespace std;
+
+
+const char* PROGRAM_NAME = "kernel-filesystem-monitor-cat";
+
+void usage(poptContext optCon, int exitcode, char *error, char *addl)
+{
+    poptPrintUsage(optCon, stderr, 0);
+    if (error) fprintf(stderr, "%s: %s0", error, addl);
+    exit(exitcode);
+}
+
+/********************************************************************************/
+/********************************************************************************/
+/********************************************************************************/
+/********************************************************************************/
+/********************************************************************************/
+/********************************************************************************/
+
+
+
+class KernelFileSystemMonitorDaemonCat
+    :
+    public KernelFileSystemMonitorDaemon
+{
+protected:
+
+    virtual void handle_event( struct inotify_event *pevent, time_t tt );
+    virtual void Closedown();
+    virtual void setupWorkingDirToPersistentDirIDMapping( long wd, const string& earl );
+
+    void
+    event_batch_start( time_t tt )
+        {}
+    
+    void
+    event_batch_end( time_t tt )
+        {}
+
+
+public:
+
+    string homedir;
+    
+    KernelFileSystemMonitorDaemonCat()
+        {}
+};
+
+
+void
+KernelFileSystemMonitorDaemonCat::setupWorkingDirToPersistentDirIDMapping(
+    long wd, const string& earl )
+{
+}
+
+void
+KernelFileSystemMonitorDaemonCat::handle_event( struct inotify_event *pevent, time_t tt )
+{
+    print_event( pevent );
+    fflush(stdout);
+}
+
+
+
+void
+KernelFileSystemMonitorDaemonCat::Closedown()
+{
+}
+
+int main( int argc, char** argv )
+{
+    unsigned long RunInForground = 0;
+    const char* homedir_CSTR     = 0;
+
+    KernelFileSystemMonitorDaemonCat* daemon
+        = new KernelFileSystemMonitorDaemonCat();
+    
+    struct poptOption optionsTable[] =
+        {
+            { "forground", 'F', POPT_ARG_NONE, &RunInForground, 0,
+              "Don't run the daemon in the background", "" },
+
+            { "homedir", 'H', POPT_ARG_STRING, &homedir_CSTR, 0,
+              "Home directory for user doing the monitoring", "" },
+
+            { 0, 0, POPT_ARG_INCLUDE_TABLE, daemon->getPopTable(), \
+              0, "generic kfsmd daemon options:", 0 },
+
+            { "verbose", 'v', POPT_ARG_NONE, &Verbose, 0,
+              "output more info", "" },
+            
+            
+            POPT_AUTOHELP
+            POPT_TABLEEND
+        };
+    poptContext optCon;
+
+    optCon = poptGetContext(PROGRAM_NAME, argc, (const char**)argv, optionsTable, 0);
+    poptSetOtherOptionHelp(optCon, "[OPTIONS]* [IGNOREPFX URL]* [WATCH URL]+ ...");
+
+    
+    /* Now do options processing */
+    char c=-1;
+    while ((c = poptGetNextOpt(optCon)) >= 0)
+    {
+    }
+
+    daemon->ParseWatchOptions( optCon );
+    daemon->setRunInForground( true );
+    daemon->setupSignalHandlers();
+
+    daemon->homedir = getHomeDir( homedir_CSTR );
+
+    try
+    {
+        if( Verbose )
+            cerr << "setting up watches" << endl;
+        daemon->setupWatches();
+        if( Verbose )
+            cerr << "calling run" << endl;
+        return daemon->run();
+    }
+    catch( exception& e )
+    {
+        cerr << "Exiting due to error reason:" << e.what() << endl;
+    }
+    
+    return 0;
+}
+