cornflake 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YjAyM2YwYzNkY2RiMDlmOTZkNjVkZTZiYmE1MWRlYjBjNzM2ODAyYg==
+  data.tar.gz: !binary |-
+    MjhkNjRhZTYxYjNhY2IxZjZlNTdmZmZlYTUyY2FlZmE0NGZiMzc5Ng==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZDRlNjJhMWQ2ODY1YTgyNTQ4ZGUxNTBmOTVjMjlmZjZjYjkxN2ZjZjI4NThk
+    ZWIxYzY1MGY1YWU0YzczMjM1NTFhMjQzYjhlYThhMWQ4ZGMyZjYzNjAxY2Zj
+    NTA5ZGRjZTdlY2M4ZTVmZmRlMThmMDAyOTlkZjUwMzk1Mjk3YTk=
+  data.tar.gz: !binary |-
+    MmIzNDEzNjkxNWFmMTZmMDQxMjU1MzU1YjUwZDI3Mzk3NGY1YmFmMTNkNjJl
+    YjNkNjM5OTMwMDkwM2Q5ZWUxMDkzNjQxMjIxMDRhZDE3N2FmM2E5NzE5Mzkw
+    YWEwNzIxN2ZjMWEyMzk2MDYxMzFmNTk0YzVjZjA4MzhkNTA2ODg=

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org/"
+gemspec
+
+gem "rake-compiler"
+gem "minitest"

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    cornflake (0.0.1)
+      macaddr (~> 1.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    macaddr (1.6.1)
+      systemu (~> 2.5.0)
+    minitest (5.0.6)
+    rake (10.1.0)
+    rake-compiler (0.9.1)
+      rake
+    systemu (2.5.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cornflake!
+  minitest
+  rake-compiler

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/clean'
+require 'rake/extensiontask'
+
+# Ruby Extension
+Rake::ExtensionTask.new('cornflake')
+
+# Packaging
+require 'bundler/gem_tasks'

data/bin/Makefile

@@ -0,0 +1,6 @@
+cornflake: ../ext/cornflake/cornflake.c
+	gcc -O2 -Wno-unused-result \
+		-DCORNFLAKE_STANDALONE \
+		../ext/cornflake/cornflake.c \
+		-lpthread \
+		-o cornflake

data/bin/race.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+for ((i=0;i<15;i++))
+do
+	./cornflake -n50000 > out.$i.log &
+done
+
+wait
+
+rm -f out.uniq
+
+for ((i=0;i<15;i++))
+do
+	cat out.$i.log >> out.uniq.log
+done
+
+wc -l out.uniq.log
+sort out.uniq.log | uniq -d
+rm -rf out.*.log
+

data/cornflake.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |s|
+  s.name = 'cornflake'
+  s.version = '0.0.1'
+  s.authors = ["Vicent Martí"]
+  s.date = Time.now.utc.strftime("%Y-%m-%d")
+  s.email = %q{vmg@github.com}
+  s.extensions = ["ext/cornflake/extconf.rb"]
+  s.files = `git ls-files`.split("\n")
+  s.homepage = %q{http://github.com/github/cornflake}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.4.2}
+  s.summary = %q{K-ordered id generation, brought to you by Unix (TM)}
+  s.description = "This does the thing with the ID generation and all that"
+  s.license = "MIT"
+
+  s.add_dependency "macaddr", "~> 1.6"
+end

data/ext/cornflake/cornflake.c

@@ -0,0 +1,387 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+
+#include "cornflake.h"
+
+#define cpu_relax() __asm__ volatile("pause\n": : :"memory")
+
+typedef uint32_t spinlock_t;
+
+static inline int spin_trylock(spinlock_t *lock)
+{
+	size_t tries = 1000;
+
+	while (tries--) {
+		if (!__sync_lock_test_and_set(lock, 1))
+			return 0;
+
+		while (*lock) {
+			cpu_relax();
+		}
+	}
+
+	return -1;
+}
+
+static inline int spin_tryunlock(spinlock_t *lock)
+{
+	if (__sync_lock_test_and_set(lock, 0) == 0)
+		return -1;
+
+	return 0;
+}
+
+static inline void spin_unlock(spinlock_t *lock)
+{
+	__sync_lock_release(lock);
+}
+
+struct cornflake {
+	spinlock_t lock;
+	uint64_t last_time;
+	uint16_t counter;
+};
+
+static uint8_t g_local_mac[6];
+
+static struct cornflake *
+cornflake_open(const char *path)
+{
+	int fd;
+	struct cornflake *corn;
+
+	fd = open(path, O_RDWR, 0666);
+	if (fd < 0)
+		return NULL;
+
+	corn = mmap(NULL, sizeof(*corn), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	close(fd);
+
+	return corn;
+}
+
+static struct cornflake *
+cornflake_create(const char *path)
+{
+	int fd;
+	struct cornflake *corn;
+
+	fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0666);
+	if (fd < 0) {
+		if (errno == EEXIST)
+			return cornflake_open(path);
+
+		return NULL;
+	}
+
+	if (ftruncate(fd, sizeof(*corn)) < 0) {
+		close(fd);
+		return NULL;
+	}
+
+	corn = mmap(NULL, sizeof(*corn), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	close(fd);
+
+	corn->lock = 0;
+	corn->last_time = 0;
+	corn->counter = 0;
+
+	return corn;
+}
+
+struct cornflake *
+cornflake_new(const char *path)
+{
+	struct cornflake *corn;
+
+	corn = cornflake_open(path);
+	if (corn)
+		return corn;
+
+	return cornflake_create(path);
+}
+
+void
+cornflake_close(struct cornflake *corn)
+{
+	munmap(corn, sizeof(*corn));
+}
+
+/**
+ * Generate a 16-byte UID. The composition of the UID is as follows.
+ *
+ * [ 8 bytes .......][ 6 bytes ...][2 bytes]
+ *  timestamp          macaddr      counter
+ *
+ *	Where:
+ *
+ *	- `timestamp` is the current time (in ms since the epoch) when the
+ *	UID was generated
+ *
+ *	- `macaddr` is the MAC address for this computer, in raw form
+ *
+ *	- `counter` is a 16-bit counter to avoid collisions between UIDs
+ *	generated in the same milisecond. This limits Cornflake to only
+ *	2^16 UID/s per milisecond.
+ *
+ *	If successful, the `uid` buffer will be filled with the unique ID
+ *	in raw form. Ensure that the buffer has 16 bytes before calling this
+ *	function.
+ *
+ *	Return values:
+ *
+ *	- CORNFLAKE_OK (0): the UID was generated successfully
+ *
+ *	- CORNFLAKE_ELOCK: there was an error when trying to lock the global
+ *	state for UID generation. This is most likely because another process
+ *	crashed while keeping the lock. As part of the error handling mechanism,
+ *	we will free the global lock, so further calls to `cornflake_gen` should
+ *	be able to work.
+ *
+ *	- CORNFLAKE_EDRIFT: the system clock has drifted into the past. Cornflake
+ *	will refuse to generate new UIDs until the clock reaches the time when the
+ *	last UID was generated, as to ensure that no UIDs are tagged in the past.
+ *
+ *	- CORNFLAKE_EOVERFLOW: the counter in the UID has overflowed (there have been
+ *	more than 2^16 unique IDs generated in this milisecond). You need to wait at
+ *	least 1 milisecond and try again.
+ */
+int
+cornflake_gen(uint8_t *uid, struct cornflake *corn)
+{
+	struct timeval tv;
+	uint64_t timestamp;
+	uint16_t sequence = 0;
+	uint8_t *bytes;
+
+	/*
+	 * We need to synchronize the global state so only *one* Cornflake
+	 * process generates UIDs simultaneously.
+	 *
+	 * We use a spinlock in shared memory (this requires x86/x64 semantics,
+	 * do not try this at home, children, specially if you have an ARM
+	 * processor at home).
+	 *
+	 * There is a caveat when locking this global state: there's an infintesimal
+	 * chance that another Cornflake process got terminated (i.e. segmentation
+	 * fault, uncaught signal) during the ~20 instructions that happen inside
+	 * this lock, before the lock has been released.
+	 *
+	 * This would usually cause every other process in the system to deadlock
+	 * while waiting for the spinlock. To work around this, we use a custom
+	 * `trylock` the spinlock: we're going to try a thousand (1000) long locks
+	 * in a row (i.e. try the exchange, if failed, loop until the lock is free,
+	 * try the exchange again).
+	 *
+	 * If we haven't acquired a lock after 1000 tries, we'll assume the spinlock
+	 * has deadlocked and we'll simply drop it and fail fast so other processes
+	 * can continue UID generations. This all should happen in less than ~500ms
+	 * even in the highest contention cases.
+	 *
+	 * FAQ: Why are you using an `xchg` spinlock instead of e.g. `pthread_mutex` on
+	 * shared memory or `pthread_spinlock`? All the `pthread` synchronization
+	 * primitives have undefined behavior when trying to unlock/destroy them from
+	 * a different thread than the one who originally acquired the lock. By controlling
+	 * the spinlock semantics, we can ensure the lock gets released when a process dies
+	 * holding it *and* we can verify that other processes do not race on the released
+	 * lock.
+	 */
+	if (spin_trylock(&corn->lock) < 0) {
+		spin_unlock(&corn->lock);
+		return CORNFLAKE_ELOCK;
+	}
+
+	/*
+	 * Get the current time in ms -- this will be the timestamp that shows
+	 * on the first 8 bytes of the UID
+	 */
+	gettimeofday(&tv, NULL);
+	timestamp = ((uint64_t)tv.tv_sec * 1000) +
+		((uint64_t)tv.tv_usec / 1000);
+
+	/*
+	 * If the current timestamp is behind the timestamp of the last UID we've
+	 * generated, that means our system clock has drifted. We cannot generate
+	 * new UIDs until the clock is back on time.
+	 */
+	if (timestamp < corn->last_time) {
+		spin_unlock(&corn->lock);
+		return CORNFLAKE_EDRIFT;
+	}
+
+	/*
+	 * If the current timestamp is the same as the last UID's we've generated,
+	 * we need to increase the UID counter (last 16 bits of the UID) to prevent
+	 * collisions.
+	 *
+	 * When the UID counter wraps around (i.e. reaches 0), we've generated more
+	 * than 2^16 UIDs in the millisecond, so we need to wait to the next ms
+	 * in order to prevent collisions.
+	 */
+	if (timestamp == corn->last_time) {
+		sequence = ++corn->counter;
+		if (!sequence) {
+			spin_unlock(&corn->lock);
+			return CORNFLAKE_EOVERFLOW;
+		}
+	} else {
+		corn->last_time = timestamp;
+		corn->counter = 0;
+	}
+
+	/*
+	 * We're out of the locking zone. HOWEVER: we need to ensure that the lock
+	 * we've acquired is still acquired when we unlock it. If somehow the lock
+	 * is not taken when we try to unlock it, that means another process has
+	 * decided that we're "dead" (we took too long to generate this UID!) and
+	 * unlocked the lock for us.
+	 *
+	 * In that case, we cannot continue: the UID we've just generated could very
+	 * well be racy.
+	 */
+	if (spin_tryunlock(&corn->lock) < 0)
+		return CORNFLAKE_ELOCK;
+
+	/*
+	 * Move over all the generated data to network byte order (big endian).
+	 * This is to ensure that the UIDs are k-ordered when compared
+	 * lexicographically.
+	 */
+	bytes = (uint8_t *)&timestamp;
+	uid[0] = bytes[7];
+	uid[1] = bytes[6];
+	uid[2] = bytes[5];
+	uid[3] = bytes[4];
+	uid[4] = bytes[3];
+	uid[5] = bytes[2];
+	uid[6] = bytes[1];
+	uid[7] = bytes[0];
+
+	memcpy(&uid[8], g_local_mac, sizeof(g_local_mac));
+
+	bytes = (uint8_t *)&sequence;
+	uid[14] = bytes[1];
+	uid[15] = bytes[0];
+
+	return 0;
+}
+
+int
+cornflake_gen_hex(char *hex_uid, size_t len, struct cornflake *corn)
+{
+	static char to_hex[] = "0123456789abcdef";
+
+	uint8_t uid[16];
+	int i, ret;
+	
+	if (len < 33)
+		return CORNFLAKE_EOVERFLOW;
+
+	ret = cornflake_gen(uid, corn);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < 16; ++i) {
+		*hex_uid++ = to_hex[uid[i] >> 4];
+		*hex_uid++ = to_hex[uid[i] & 0xf];
+	}
+
+	*hex_uid = '\0';
+	return 0;
+}
+
+int
+cornflake_set_mac(const char *mac_addr)
+{
+	if (strlen(mac_addr) != 17)
+		return CORNFLAKE_EMACADDR;
+
+	if (sscanf(mac_addr, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
+		&g_local_mac[0], &g_local_mac[1], &g_local_mac[2],
+		&g_local_mac[3], &g_local_mac[4], &g_local_mac[5]) != 6)
+		return CORNFLAKE_EMACADDR;
+
+	return 0;
+}
+
+#ifdef CORNFLAKE_STANDALONE
+#include "getopt.h"
+
+static inline double bench_time(void)
+{
+	struct timespec ts;
+	clock_gettime(CLOCK_REALTIME, &ts);
+	return (ts.tv_sec * 1000.0) + (ts.tv_nsec / 1000000.0);
+}
+
+
+int main(int argc, char *argv[])
+{
+	static struct option longopts[] = {
+		{ "count", required_argument, NULL, 'n' },
+		{ "path", required_argument, NULL, 'o' },
+		{ "time", no_argument, NULL,	't' },
+		{ "version", no_argument, NULL,	'v' },
+		{ NULL,  0,  NULL, 0 }
+	};
+
+	struct cornflake *flake;
+	const char *flake_path = "./milkbowl";
+	int opt, benchmark = 0;
+	unsigned int i, count = 1;
+	double start, end;
+
+	while ((opt = getopt_long(argc, argv, ":n:o:tv", longopts, NULL)) != -1) {
+		switch (opt) {
+		case 'n': count = atoi(optarg); break;
+		case 'o': flake_path = optarg; break;
+		case 't': benchmark = 1; break;
+		case 'v':
+			puts("cornflake");
+			return 0;
+
+		default:
+			printf("Usage: %s [--path CORNFLAKE_CONTEXT] [-n COUNT]\n", argv[0]);
+			return 1;
+		}
+	}
+
+	flake = cornflake_new(flake_path);
+	if (flake == NULL) {
+		fprintf(stderr, "Failed to open Cornflake context: %s\n", flake_path);
+		return -1;
+	}
+
+	start = bench_time();
+
+	for (i = 0; i < count; ++i) {
+		char uid[33];
+		int err = cornflake_gen_hex(uid, sizeof(uid), flake);
+
+		if (err < 0) {
+			fprintf(stderr, "Generation failed: %d\n", err);
+			return -2;
+		}
+
+		puts(uid);
+	}
+
+	end = bench_time();
+
+	if (benchmark)
+		fprintf(stderr, "Benchmark: %d oids in %fms\n", count, end - start);
+
+	cornflake_close(flake);
+	return 0;
+}
+#endif

data/ext/cornflake/cornflake.h

@@ -0,0 +1,20 @@
+#ifndef __CORNFLAKE_H__
+#define __CORNFLAKE_H__
+
+struct cornflake;
+
+enum cornflake_err {
+	CORNFLAKE_OK = 0,
+	CORNFLAKE_ELOCK = -1,
+	CORNFLAKE_EDRIFT = -2,
+	CORNFLAKE_EMACADDR = -3,
+	CORNFLAKE_EOVERFLOW = -4,
+};
+
+struct cornflake *cornflake_new(const char *path);
+void cornflake_close(struct cornflake *corn);
+int cornflake_gen(uint8_t *uid_out, struct cornflake *corn);
+int cornflake_gen_hex(char *hex_uid, size_t len, struct cornflake *corn);
+int cornflake_set_mac(const char *mac_addr);
+
+#endif

data/ext/cornflake/cornflake.rb.c

@@ -0,0 +1,91 @@
+#include <ruby.h>
+#include "cornflake.h"
+
+static VALUE rb_cCornflake;
+
+static void
+rb_cornflake__free(void *corn)
+{
+	cornflake_close(corn);
+}
+
+static VALUE
+rb_cornflake__new(VALUE klass, VALUE rb_path)
+{
+	struct cornflake *corn;
+
+	Check_Type(rb_path, T_STRING);
+
+	corn = cornflake_new(StringValueCStr(rb_path));
+	if (!corn)
+		rb_raise(rb_eRuntimeError, "Failed to open Cornflake state at %s",
+			StringValueCStr(rb_path));
+
+	return Data_Wrap_Struct(klass, NULL, rb_cornflake__free, corn);
+}
+
+static void
+rb_cornflake_check(int result)
+{
+	switch (result) {
+	case CORNFLAKE_OK:
+		return; 
+
+	case CORNFLAKE_ELOCK:
+		rb_raise(rb_eRuntimeError, "Failed to lock the global Cornflake state");
+		break;
+
+	case CORNFLAKE_EDRIFT:
+		rb_raise(rb_eRuntimeError, "Computer time drifted into the past");
+		break;
+
+	default:
+		rb_raise(rb_eRuntimeError, "Failed to generate UID");
+		break;
+	}
+}
+
+static VALUE
+rb_cornflake_gen(VALUE self)
+{
+	uint8_t uid[16];
+	struct cornflake *corn;
+
+	Data_Get_Struct(self, struct cornflake, corn);
+	rb_cornflake_check(cornflake_gen(uid, corn));
+
+	return rb_str_new((char *)uid, 16);
+}
+
+static VALUE
+rb_cornflake_gen_hex(VALUE self)
+{
+	char hex_uid[33];
+	struct cornflake *corn;
+
+	Data_Get_Struct(self, struct cornflake, corn);
+	rb_cornflake_check(cornflake_gen_hex(hex_uid, sizeof(hex_uid), corn));
+
+	return rb_str_new2(hex_uid);
+}
+
+static VALUE
+rb_cornflake_set_mac(VALUE klass, VALUE rb_mac)
+{
+	Check_Type(rb_mac, T_STRING);
+
+	if (cornflake_set_mac(StringValueCStr(rb_mac)) < 0)
+		rb_raise(rb_eTypeError, "Invalid MAC Address: %s", StringValueCStr(rb_mac));
+
+	return Qnil;
+}
+
+void Init_cornflake()
+{
+	rb_cCornflake = rb_define_class("Cornflake", rb_cObject);
+	rb_define_singleton_method(rb_cCornflake, "new", rb_cornflake__new, 1);
+	rb_define_singleton_method(rb_cCornflake, "macaddr=", rb_cornflake_set_mac, 1);
+	rb_define_method(rb_cCornflake, "uid", rb_cornflake_gen, 0);
+	rb_define_method(rb_cCornflake, "hex", rb_cornflake_gen_hex, 0);
+}
+

data/ext/cornflake/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+
+dir_config('cornflake')
+create_makefile('cornflake')

data/lib/cornflake.rb

@@ -0,0 +1,26 @@
+require 'cornflake.so'
+require 'macaddr'
+
+Cornflake.macaddr = Mac.addr
+
+class Cornflake
+  module Base62
+    SIXTYTWO = ('0'..'9').to_a + ('A'..'Z').to_a + ('a'..'z').to_a
+    PADDING = 18 # 18 digits is enough to last the next 300 years
+
+    def self.encode(numeric)
+      s = ''
+      while numeric > 0
+        s << Base62::SIXTYTWO[numeric.modulo(62)]
+        numeric /= 62
+      end
+      s.reverse!
+      s.rjust(PADDING, "0")
+    end
+  end
+
+  def base62
+    Base62.encode(self.hex.to_i(16))
+  end
+end
+

data/nginx/README.md

@@ -0,0 +1,28 @@
+# Cornflake NGINX module
+
+This is a self-contained nginx module that lets you use Cornflake to generate k-ordered IDs for all requests in a system. You can use these IDs to match requests between the frontend and the backend (or whatevs, buddy) and since they are roughly sorted, you can actually tell the order (and time) of the requests.
+
+**WARNING**: This module is experimental as *fuck*. Do not deploy on production yet. It needs a lot more testing.
+
+## Usage
+
+1. Add `--add-module=../cornflake/nginx` to your nginx configure command. `make` dat sexy daemon. That will build the cornflake module.
+
+2. Now you have a `$request_id` variable. Do things with it. E.g.
+
+    ```
+    server {
+        listen       80;
+        server_name  github.com;
+        location / {
+            proxy_set_header x-github-cornflake-id $request_id;
+            proxy_pass   http://localhost:8080;
+        }
+    }
+    ```
+
+    Pick it up on the backend. Or whatever you want. Unclear.
+
+## Credits
+
+lol u know who wrote tis thang

data/nginx/cornflake.c

@@ -0,0 +1,387 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+
+#include "cornflake.h"
+
+#define cpu_relax() __asm__ volatile("pause\n": : :"memory")
+
+typedef uint32_t spinlock_t;
+
+static inline int spin_trylock(spinlock_t *lock)
+{
+	size_t tries = 1000;
+
+	while (tries--) {
+		if (!__sync_lock_test_and_set(lock, 1))
+			return 0;
+
+		while (*lock) {
+			cpu_relax();
+		}
+	}
+
+	return -1;
+}
+
+static inline int spin_tryunlock(spinlock_t *lock)
+{
+	if (__sync_lock_test_and_set(lock, 0) == 0)
+		return -1;
+
+	return 0;
+}
+
+static inline void spin_unlock(spinlock_t *lock)
+{
+	__sync_lock_release(lock);
+}
+
+struct cornflake {
+	spinlock_t lock;
+	uint64_t last_time;
+	uint16_t counter;
+};
+
+static uint8_t g_local_mac[6];
+
+static struct cornflake *
+cornflake_open(const char *path)
+{
+	int fd;
+	struct cornflake *corn;
+
+	fd = open(path, O_RDWR, 0666);
+	if (fd < 0)
+		return NULL;
+
+	corn = mmap(NULL, sizeof(*corn), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	close(fd);
+
+	return corn;
+}
+
+static struct cornflake *
+cornflake_create(const char *path)
+{
+	int fd;
+	struct cornflake *corn;
+
+	fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0666);
+	if (fd < 0) {
+		if (errno == EEXIST)
+			return cornflake_open(path);
+
+		return NULL;
+	}
+
+	if (ftruncate(fd, sizeof(*corn)) < 0) {
+		close(fd);
+		return NULL;
+	}
+
+	corn = mmap(NULL, sizeof(*corn), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	close(fd);
+
+	corn->lock = 0;
+	corn->last_time = 0;
+	corn->counter = 0;
+
+	return corn;
+}
+
+struct cornflake *
+cornflake_new(const char *path)
+{
+	struct cornflake *corn;
+
+	corn = cornflake_open(path);
+	if (corn)
+		return corn;
+
+	return cornflake_create(path);
+}
+
+void
+cornflake_close(struct cornflake *corn)
+{
+	munmap(corn, sizeof(*corn));
+}
+
+/**
+ * Generate a 16-byte UID. The composition of the UID is as follows.
+ *
+ * [ 8 bytes .......][ 6 bytes ...][2 bytes]
+ *  timestamp          macaddr      counter
+ *
+ *	Where:
+ *
+ *	- `timestamp` is the current time (in ms since the epoch) when the
+ *	UID was generated
+ *
+ *	- `macaddr` is the MAC address for this computer, in raw form
+ *
+ *	- `counter` is a 16-bit counter to avoid collisions between UIDs
+ *	generated in the same milisecond. This limits Cornflake to only
+ *	2^16 UID/s per milisecond.
+ *
+ *	If successful, the `uid` buffer will be filled with the unique ID
+ *	in raw form. Ensure that the buffer has 16 bytes before calling this
+ *	function.
+ *
+ *	Return values:
+ *
+ *	- CORNFLAKE_OK (0): the UID was generated successfully
+ *
+ *	- CORNFLAKE_ELOCK: there was an error when trying to lock the global
+ *	state for UID generation. This is most likely because another process
+ *	crashed while keeping the lock. As part of the error handling mechanism,
+ *	we will free the global lock, so further calls to `cornflake_gen` should
+ *	be able to work.
+ *
+ *	- CORNFLAKE_EDRIFT: the system clock has drifted into the past. Cornflake
+ *	will refuse to generate new UIDs until the clock reaches the time when the
+ *	last UID was generated, as to ensure that no UIDs are tagged in the past.
+ *
+ *	- CORNFLAKE_EOVERFLOW: the counter in the UID has overflowed (there have been
+ *	more than 2^16 unique IDs generated in this milisecond). You need to wait at
+ *	least 1 milisecond and try again.
+ */
+int
+cornflake_gen(uint8_t *uid, struct cornflake *corn)
+{
+	struct timeval tv;
+	uint64_t timestamp;
+	uint16_t sequence = 0;
+	uint8_t *bytes;
+
+	/*
+	 * We need to synchronize the global state so only *one* Cornflake
+	 * process generates UIDs simultaneously.
+	 *
+	 * We use a spinlock in shared memory (this requires x86/x64 semantics,
+	 * do not try this at home, children, specially if you have an ARM
+	 * processor at home).
+	 *
+	 * There is a caveat when locking this global state: there's an infintesimal
+	 * chance that another Cornflake process got terminated (i.e. segmentation
+	 * fault, uncaught signal) during the ~20 instructions that happen inside
+	 * this lock, before the lock has been released.
+	 *
+	 * This would usually cause every other process in the system to deadlock
+	 * while waiting for the spinlock. To work around this, we use a custom
+	 * `trylock` the spinlock: we're going to try a thousand (1000) long locks
+	 * in a row (i.e. try the exchange, if failed, loop until the lock is free,
+	 * try the exchange again).
+	 *
+	 * If we haven't acquired a lock after 1000 tries, we'll assume the spinlock
+	 * has deadlocked and we'll simply drop it and fail fast so other processes
+	 * can continue UID generations. This all should happen in less than ~500ms
+	 * even in the highest contention cases.
+	 *
+	 * FAQ: Why are you using an `xchg` spinlock instead of e.g. `pthread_mutex` on
+	 * shared memory or `pthread_spinlock`? All the `pthread` synchronization
+	 * primitives have undefined behavior when trying to unlock/destroy them from
+	 * a different thread than the one who originally acquired the lock. By controlling
+	 * the spinlock semantics, we can ensure the lock gets released when a process dies
+	 * holding it *and* we can verify that other processes do not race on the released
+	 * lock.
+	 */
+	if (spin_trylock(&corn->lock) < 0) {
+		spin_unlock(&corn->lock);
+		return CORNFLAKE_ELOCK;
+	}
+
+	/*
+	 * Get the current time in ms -- this will be the timestamp that shows
+	 * on the first 8 bytes of the UID
+	 */
+	gettimeofday(&tv, NULL);
+	timestamp = ((uint64_t)tv.tv_sec * 1000) +
+		((uint64_t)tv.tv_usec / 1000);
+
+	/*
+	 * If the current timestamp is behind the timestamp of the last UID we've
+	 * generated, that means our system clock has drifted. We cannot generate
+	 * new UIDs until the clock is back on time.
+	 */
+	if (timestamp < corn->last_time) {
+		spin_unlock(&corn->lock);
+		return CORNFLAKE_EDRIFT;
+	}
+
+	/*
+	 * If the current timestamp is the same as the last UID's we've generated,
+	 * we need to increase the UID counter (last 16 bits of the UID) to prevent
+	 * collisions.
+	 *
+	 * When the UID counter wraps around (i.e. reaches 0), we've generated more
+	 * than 2^16 UIDs in the millisecond, so we need to wait to the next ms
+	 * in order to prevent collisions.
+	 */
+	if (timestamp == corn->last_time) {
+		sequence = ++corn->counter;
+		if (!sequence) {
+			spin_unlock(&corn->lock);
+			return CORNFLAKE_EOVERFLOW;
+		}
+	} else {
+		corn->last_time = timestamp;
+		corn->counter = 0;
+	}
+
+	/*
+	 * We're out of the locking zone. HOWEVER: we need to ensure that the lock
+	 * we've acquired is still acquired when we unlock it. If somehow the lock
+	 * is not taken when we try to unlock it, that means another process has
+	 * decided that we're "dead" (we took too long to generate this UID!) and
+	 * unlocked the lock for us.
+	 *
+	 * In that case, we cannot continue: the UID we've just generated could very
+	 * well be racy.
+	 */
+	if (spin_tryunlock(&corn->lock) < 0)
+		return CORNFLAKE_ELOCK;
+
+	/*
+	 * Move over all the generated data to network byte order (big endian).
+	 * This is to ensure that the UIDs are k-ordered when compared
+	 * lexicographically.
+	 */
+	bytes = (uint8_t *)&timestamp;
+	uid[0] = bytes[7];
+	uid[1] = bytes[6];
+	uid[2] = bytes[5];
+	uid[3] = bytes[4];
+	uid[4] = bytes[3];
+	uid[5] = bytes[2];
+	uid[6] = bytes[1];
+	uid[7] = bytes[0];
+
+	memcpy(&uid[8], g_local_mac, sizeof(g_local_mac));
+
+	bytes = (uint8_t *)&sequence;
+	uid[14] = bytes[1];
+	uid[15] = bytes[0];
+
+	return 0;
+}
+
+int
+cornflake_gen_hex(char *hex_uid, size_t len, struct cornflake *corn)
+{
+	static char to_hex[] = "0123456789abcdef";
+
+	uint8_t uid[16];
+	int i, ret;
+	
+	if (len < 33)
+		return CORNFLAKE_EOVERFLOW;
+
+	ret = cornflake_gen(uid, corn);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < 16; ++i) {
+		*hex_uid++ = to_hex[uid[i] >> 4];
+		*hex_uid++ = to_hex[uid[i] & 0xf];
+	}
+
+	*hex_uid = '\0';
+	return 0;
+}
+
+int
+cornflake_set_mac(const char *mac_addr)
+{
+	if (strlen(mac_addr) != 17)
+		return CORNFLAKE_EMACADDR;
+
+	if (sscanf(mac_addr, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
+		&g_local_mac[0], &g_local_mac[1], &g_local_mac[2],
+		&g_local_mac[3], &g_local_mac[4], &g_local_mac[5]) != 6)
+		return CORNFLAKE_EMACADDR;
+
+	return 0;
+}
+
+#ifdef CORNFLAKE_STANDALONE
+#include "getopt.h"
+
+static inline double bench_time(void)
+{
+	struct timespec ts;
+	clock_gettime(CLOCK_REALTIME, &ts);
+	return (ts.tv_sec * 1000.0) + (ts.tv_nsec / 1000000.0);
+}
+
+
+int main(int argc, char *argv[])
+{
+	static struct option longopts[] = {
+		{ "count", required_argument, NULL, 'n' },
+		{ "path", required_argument, NULL, 'o' },
+		{ "time", no_argument, NULL,	't' },
+		{ "version", no_argument, NULL,	'v' },
+		{ NULL,  0,  NULL, 0 }
+	};
+
+	struct cornflake *flake;
+	const char *flake_path = "./milkbowl";
+	int opt, benchmark = 0;
+	unsigned int i, count = 1;
+	double start, end;
+
+	while ((opt = getopt_long(argc, argv, ":n:o:tv", longopts, NULL)) != -1) {
+		switch (opt) {
+		case 'n': count = atoi(optarg); break;
+		case 'o': flake_path = optarg; break;
+		case 't': benchmark = 1; break;
+		case 'v':
+			puts("cornflake");
+			return 0;
+
+		default:
+			printf("Usage: %s [--path CORNFLAKE_CONTEXT] [-n COUNT]\n", argv[0]);
+			return 1;
+		}
+	}
+
+	flake = cornflake_new(flake_path);
+	if (flake == NULL) {
+		fprintf(stderr, "Failed to open Cornflake context: %s\n", flake_path);
+		return -1;
+	}
+
+	start = bench_time();
+
+	for (i = 0; i < count; ++i) {
+		char uid[33];
+		int err = cornflake_gen_hex(uid, sizeof(uid), flake);
+
+		if (err < 0) {
+			fprintf(stderr, "Generation failed: %d\n", err);
+			return -2;
+		}
+
+		puts(uid);
+	}
+
+	end = bench_time();
+
+	if (benchmark)
+		fprintf(stderr, "Benchmark: %d oids in %fms\n", count, end - start);
+
+	cornflake_close(flake);
+	return 0;
+}
+#endif

data/nginx/cornflake.h

@@ -0,0 +1,20 @@
+#ifndef __CORNFLAKE_H__
+#define __CORNFLAKE_H__
+
+struct cornflake;
+
+enum cornflake_err {
+	CORNFLAKE_OK = 0,
+	CORNFLAKE_ELOCK = -1,
+	CORNFLAKE_EDRIFT = -2,
+	CORNFLAKE_EMACADDR = -3,
+	CORNFLAKE_EOVERFLOW = -4,
+};
+
+struct cornflake *cornflake_new(const char *path);
+void cornflake_close(struct cornflake *corn);
+int cornflake_gen(uint8_t *uid_out, struct cornflake *corn);
+int cornflake_gen_hex(char *hex_uid, size_t len, struct cornflake *corn);
+int cornflake_set_mac(const char *mac_addr);
+
+#endif

data/nginx/ngx_cornflake_module.c

@@ -0,0 +1,80 @@
+#include <ngx_core.h>
+#include <ngx_http.h>
+#include <nginx.h>
+#include <ngx_http_variables.h>
+#include "cornflake.h"
+
+static struct cornflake *g_cornflake;
+
+ngx_int_t
+ngx_cornflake_get_var(ngx_http_request_t *r,
+	ngx_http_variable_value_t *v, uintptr_t data)
+{
+	char *uid = ngx_pnalloc(r->pool, 33);
+
+	if (uid == NULL)
+		return NGX_ERROR;
+
+	if (cornflake_gen_hex(uid, 33, g_cornflake) < 0) {
+		ngx_pfree(r->pool, uid);
+		return NGX_ERROR;
+	}
+
+	v->len = 32;
+	v->valid = 1;
+	v->no_cacheable = 0;
+	v->not_found = 0;
+	v->data = (u_char *)uid;
+
+	return NGX_OK;
+}
+
+static ngx_int_t add_variables(ngx_conf_t *cf)
+{
+	static ngx_str_t var_name = ngx_string("request_id");
+	ngx_http_variable_t *var = NULL;
+
+	g_cornflake = cornflake_new("/etc/cornflake");
+	if (g_cornflake == NULL)
+		return NGX_ERROR;
+
+	var = ngx_http_add_variable(cf, &var_name, NGX_HTTP_VAR_NOHASH);
+	if (var == NULL)
+		return NGX_ERROR;
+
+	var->get_handler = ngx_cornflake_get_var;
+	return NGX_OK;
+}
+
+static ngx_http_module_t  cornflake_context = {
+	add_variables,  /* preconfiguration */
+	NULL,			/* postconfiguration */
+
+	NULL,        /* create main configuration */
+	NULL,        /* init main configuration */
+
+	NULL,        /* create server configuration */
+	NULL,        /* merge server configuration */
+
+	NULL,        /* create location configuration */
+	NULL         /* merge location configuration */
+};
+
+static ngx_command_t cornflake_commands[] = {
+	ngx_null_command
+};
+
+ngx_module_t ngx_cornflake_module = {
+	NGX_MODULE_V1,
+	&cornflake_context,		/* module context */
+	cornflake_commands,	/* module directives */
+	NGX_HTTP_MODULE,		/* module type */
+	NULL,                              /* init master */
+	NULL,                              /* init module */
+	NULL,                              /* init process */
+	NULL,                              /* init thread */
+	NULL,                              /* exit thread */
+	NULL,                              /* exit process */
+	NULL,                              /* exit master */
+	NGX_MODULE_V1_PADDING
+};

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: cornflake
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Vicent Martí
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: macaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: This does the thing with the ID generation and all that
+email: vmg@github.com
+executables: []
+extensions:
+- ext/cornflake/extconf.rb
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- bin/Makefile
+- bin/race.sh
+- cornflake.gemspec
+- ext/cornflake/cornflake.c
+- ext/cornflake/cornflake.h
+- ext/cornflake/cornflake.rb.c
+- ext/cornflake/extconf.rb
+- lib/cornflake.rb
+- nginx/README.md
+- nginx/cornflake.c
+- nginx/cornflake.h
+- nginx/ngx_cornflake_module.c
+homepage: http://github.com/github/cornflake
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: K-ordered id generation, brought to you by Unix (TM)
+test_files: []