omgf 0.0.0.GIT

data/GIT-VERSION-GEN

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+GVF=GIT-VERSION-FILE
+DEF_VER=v0.0.0.GIT
+
+LF='
+'
+
+# First see if there is a version file (included in release tarballs),
+# then try git-describe, then default.
+if test -f version
+then
+	VN=$(cat version) || VN="$DEF_VER"
+elif test -d .git -o -f .git &&
+	VN=$(git describe --abbrev=4 HEAD 2>/dev/null) &&
+	case "$VN" in
+	*$LF*) (exit 1) ;;
+	v[0-9]*)
+		git update-index -q --refresh
+		test -z "$(git diff-index --name-only HEAD --)" ||
+		VN="$VN-dirty" ;;
+	esac
+then
+	VN=$(echo "$VN" | sed -e 's/-/./g');
+else
+	VN="$DEF_VER"
+fi
+
+VN=$(expr "$VN" : v*'\(.*\)')
+
+if test -r $GVF
+then
+	VC=$(sed -e 's/^GIT_VERSION = //' <$GVF)
+else
+	VC=unset
+fi
+test "$VN" = "$VC" || {
+	echo >&2 "GIT_VERSION = $VN"
+	echo "GIT_VERSION = $VN" >$GVF
+}

data/GNUmakefile

@@ -0,0 +1,5 @@
+all::
+RSYNC_DEST := bogomips.org:/srv/bogomips/omgf
+rfproject := rainbows
+rfpackage := omgf
+include pkg.mk

data/README

@@ -0,0 +1,69 @@
+= OMGF - hysterical REST API for MogileFS using Rack
+
+OMGF provides an HTTP interface for {MogileFS}[http://mogilefs.org],
+allowing clients to connect to MogileFS without needing specialized
+libraries to interface with MogileFS.  Currently, WebDAV is supported.
+OMGF is short for "OMGFiles", an anagram for "MogileFS".
+
+It is built on top of the
+{Ruby mogilefs-client}[http://bogomips.org/mogilefs-client] library
+
+== Features
+
+* Can be used unify multiple, independent MogileFS installations
+  into a single HTTP endpoint.
+
+== Install
+
+OMGF is only compatible with Ruby 1.9.3 and later.
+
+You may install it via RubyGems on RubyGems.org:
+
+  gem install omgf
+
+== Usage
+
+ TODO...
+
+== Development
+
+You can get the latest source via git from the following locations
+(these versions may not be stable):
+
+  git://bogomips.org/omgf.git
+  git://repo.or.cz/omgf.git (mirror)
+
+You may browse the code from the web and download the latest snapshot
+tarballs here:
+
+== Disclaimer
+
+There is NO WARRANTY whatsoever if anything goes wrong, but let us know
+and we'll try our best to fix it.
+
+All feedback (bug reports, user/development dicussion, patches, pull
+requests) go to the mailing list/newsgroup.  See the ISSUES document for
+information on the {omgf mailing list}[mailto:omgf@librelist.org]
+
+You're can (and probably should) restrict access to OMGF hosts/ports.
+Anybody who can access OMGF must also be able to access the source code
+in accordance with the AGPL.  OMGF is not written/audited by a security
+expert, so it is likely not secure and you should only give trusted
+users access to it.
+
+== License
+
+OMGF is copyrighted Free Software by all contributors, see logs in
+revision control for names and email addresses of all of them.
+
+You can redistribute OMGF and/or modify it under the terms of the GNU
+Affero General Public License, version 3 or later as published by the
+Free Software Foundation.
+
+OMGF is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with OMGF; if not, see https://www.gnu.org/licenses

data/examples/hyst.README

@@ -0,0 +1,90 @@
+NAME
+  hyst - command-line client for OMGF::HystericalRaisins REST API
+SYNOPSIS
+  hyst <ls|cat|zcat|get|put|tee|stat|rm|wait-repl>
+DESCRIPTION
+  A simple command-line client for the OMGF::HystericalRaisins REST API
+
+COMMANDS
+
+  ls [-l -h -a -A<AFTER>] [<PREFIX>]
+    - list keys, matching optional PREFIX
+
+    Options:
+      -l         show long listing format: "devcount  size  key"
+      -h         with -l, show sizes in human-readable format
+      -a         list MogileFS-specific metadata keys as well
+      -A<AFTER>  list keys that are sorted (ASCII-betically) after
+                 the given argument
+
+  cat [-r<RANGE>] <KEY>
+    - write contents of KEY to stdout
+
+    Options:
+      -r<RANGE>  show only the given byte range, see the curl(1)
+                 documentation for more details
+
+  zcat <KEY>
+    - pipe contents of KEY to (g)zcat and write the result to stdout
+
+  get [-r<RANGE>] <KEY>
+    - download contents of KEY into the basename of KEY
+
+    Options:
+      -r<RANGE>  show only the given byte range, see the curl(1)
+                 documentation for more details
+
+  cp <FILE> <KEY>
+    - copy the contents of FILE into KEY
+
+  put [-f] <KEY>
+    - copy the contents of KEY (on the local FS) into KEY
+
+    Options:
+      -f         overwrite existing keys
+
+  tee [-f] <KEY>
+    - read from stdin and write to both KEY and stdout
+
+    Options:
+      -f         overwrite existing keys
+
+  stat <KEY1> [<KEY2> ...]
+    - show metadata in an RFC822-like format
+      Currently, `Key', `Size', and `URL-$nr' keys are shown
+
+  rm <KEY1> [<KEY2> ...]
+    - delete given keys
+
+  wait-repl [-t<TIMEOUT> -n<COUNT>] <KEY>
+    - poll the HystericalRaisins server until the replication count of KEY has
+      reached the given COUNT.  Will exit with status code 1 on
+      failure after TIMEOUT seconds and 0 on success if the desired
+      replication count is reached.
+
+    Options:
+      -n<COUNT>    Wait until there are COUNT copies of KEY.
+                   Default: 2 copies
+      -t<TIMEOUT>  exit with failure after TIMEOUT seconds have passed
+                   and COUNT is not reached.  Default is 3600 seconds.
+
+ENVIRONMENT
+
+  MOG_DOMAIN must be set to your MogileFS namespace/domain
+
+  HYST_HOST needs to be set to point to the HOST:PORT of your
+  HystericalRaisins server.
+
+  HYST_CURL_OPTS specifies command-line options `hyst' passes to
+  curl(1).  Default: --silent --show-error --fail --no-buffer
+
+FILES
+
+  ~/.hystrc - a Bourne-Again SHell shell snippet for sourcing environment
+              variables (e.g. HYST_HOST, HYST_CURL_OPTS)
+
+SEE ALSO
+  curl(1)
+
+AUTHOR
+  Eric Wong <normalperson@yhbt.net>

data/examples/hyst.bash

@@ -0,0 +1,365 @@
+#!/bin/bash
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+
+set -e
+# bash v3+ is required for pipefail
+set -o pipefail
+
+# standard environment, hyst only supports printable URL-safe characters
+# in the PATH anyways and payload data is binary (like any other UNIX fs).
+unset CDPATH
+export LC_ALL=C LANG=C
+die() {
+	echo >&2 "$@"
+	exit 1
+}
+
+if test -f ~/.hystrc
+then
+	. ~/.hystrc
+fi
+
+test -n "$HYST_HOST" || die "E: HYST_HOST must be set in the environment"
+test -n "$MOG_DOMAIN" || die 'E: MOG_DOMAIN must be set in environment'
+
+base=http://$HYST_HOST/$MOG_DOMAIN
+valid_ends='a-zA-Z0-9_-'
+valid="./$valid_ends"
+self="$0"
+usage="Usage: $self <ls|cat|zcat|get|put|tee|stat|rm|wait-repl>"
+if test x"$1" = x--help; then die "$usage"; fi
+
+# inline awk scripts:
+humanize_begin='BEGIN{ U[0]="K"; U[1]="M"; U[2]="G" }'
+humanize_epl='
+if ($2 > 1024) {
+	suff = "";
+	for (i = 0; i < 3; i++) {
+		$2 /= 1024.0;
+		suff = U[i];
+		if ($2 < 1024)
+			break;
+	}
+	$2 = sprintf("% 10.1f%s", $2, suff);
+}
+'
+stat_fmt='
+$1 == "Content-Length:" {
+	size = $2;
+	sub(/\r/, "", size);
+}
+$1 ~ /^X-Url-[0-9]+:$/ {
+	id = $1;
+	u = $2;
+	sub(/X-Url-/, "", id);
+	sub(/:/, "", id);
+	sub(/\r/, "", u);
+
+	url[id] = u;
+	++nr_url;
+}
+END {
+	if (NR == 0) {
+		print "Cannot stat key:", key > "/dev/stderr"
+	} else {
+		print "Key:", key;
+		print "Size:", size;
+		for(i = 0; i < nr_url; ++i)
+			printf "URL-%d: %s\n", i, url[i];
+		exit(nr_url == 0 ? 1 : 0);
+	}
+}
+'
+stdout_is_null='
+/\/dev\/null$/ {
+	null_Mm = $5""$6;
+}
+/\/fd\/1$/ {
+	fd_1_Mm = $5""$6;
+}
+END {
+	if (NR == 2 && fd_1_Mm == null_Mm)
+		print "t"
+}
+'
+
+run_curl () {
+	o='--silent --show-error --fail --no-buffer'
+	curl_opts=${HYST_CURL_OPTS-$o}
+	# echo >&2 T: curl $curl_opts $v "$@"
+	curl $curl_opts $v "$@"
+}
+
+invalid_key () {
+	echo >&2 "E: key '$1' is invalid"
+	echo >&2 "E: only the following characters are allowed: $valid"
+	echo >&2 "E: redundant slashes are not allowed"
+	die "E: (no leading or trailing slash '/' or period '.')"
+}
+
+parse_common() {
+	opts="-v $1"; shift
+	key_args="$1"; shift
+	usage="Usage: $self $command [$opts] $key_args"
+	if test x"$1" = x--help; then die "$usage"; fi
+	optstr=
+	req_arg="$(expr x"$key_args" : x'\(<[A-Z0-9]*>\).*$' || :)"
+	one_arg="$(expr x"$key_args" : x'\(<[A-Z0-9]*>\)$' || :)"
+	file_arg="$(expr x"$key_args" : x'\(<FILE>\)' || :)"
+
+	f= l= h= file= keys= ok= key= v= o_bool= o_arg=
+
+	for b in $opts
+	do
+		o=$(expr x"$b" : 'x-\([A-Za-z]\)$' || :)
+		case $o in
+		'')
+			o=$(expr x"$b" : 'x-\([A-Za-z]\)<[^>]*>$')
+			o_arg="$o_arg$o"
+			optstr="$optstr$o:"
+			;;
+		*)
+			o_bool="$o_bool$o"
+			optstr="$optstr$o"
+			;;
+		esac
+	done
+
+	opts=$(getopt $optstr $*)
+	set -- $opts
+	while test $# -ne 0
+	do
+		i=$1; shift
+		case $i in
+		-[$o_bool])
+			_tmp=$(expr x$i : 'x-\(['$o_bool']\)')
+			eval $_tmp=$i
+			;;
+		-[$o_arg])
+			_tmp=$(expr x$i : 'x-\(['$o_arg']\)')
+			eval $_tmp=$i"$1"
+			shift
+			;;
+		--)
+			case $command in
+			ls)
+				ok1='\(['$valid']*\)$'
+				ok2="$ok1"
+				;;
+			*)
+				ok1='\(['$valid_ends']['$valid']*\)$'
+				ok2='\(['$valid']*['$valid_ends']\)$'
+				;;
+			esac
+			;;
+		*)
+			if test -n "$one_arg" && test -n "$keys"
+			then
+				die "$usage"
+			fi
+
+			if test -n "$file_arg" && test -z "$file"
+			then
+				file="$i"
+			else
+				case $i in
+				*//*) invalid_key "$i" ;;
+				esac
+
+				k1=$(expr x"$i" : x"$ok1" || :)
+				k2=$(expr x"$i" : x"$ok2" || :)
+				if test x"$k1" != x"$k2" || test -z "$k1"
+				then
+					invalid_key "$i"
+				fi
+				keys="$keys $k1"
+				if test -z "$key"; then key="$k1"; fi
+			fi
+			;;
+		esac
+	done
+
+	if test -n "$req_arg"
+	then
+		test -n "$keys" || die "$usage"
+	fi
+
+	force=
+	if test -n "$f"
+	then
+		force="-H X-OMGF-Force:true"
+	fi
+}
+
+keys_to_urls() {
+	pfx="$1"
+	urls=
+	for i in $keys; do urls="$urls $pfx $base/$i"; done
+}
+
+# run!
+command="$1"
+test -n "$command" || die "$usage"
+shift
+case $command in
+ls)
+	parse_common '-l -h -a -A<AFTER> -B<BEFORE>' '[<PREFIX>]' "$@"
+	limit=1000 last= prefix="$key" begin=
+	epl='print $1'
+	query="?limit=$limit"
+
+	if test -n "$B"
+	then
+		before="$(expr x"$B" : 'x-B\(.*\)$')"
+		before='$1 >= "'$before'" { exit 0 }'
+	fi
+	case "$B,$a" in
+	,) match= ;;
+	,-a) match= ;;
+	*,-a) match="$before" ;;
+	*,) match="$before" ;;
+	esac
+
+	if test -n "$l"
+	then
+		epl='printf "% 2d % 16s     %s\n", $3, $2, $1;'
+		if test -n "$h"
+		then
+			epl="$humanize_epl$epl"
+			begin="$humanize_begin"
+		fi
+	fi
+
+	test -z "$prefix" || query="$query&prefix=$prefix"
+	a="$begin$match{$epl}"'END {print "last="$1, "nr="NR >"/dev/stderr"}'
+
+	q="$query"
+	if test -n "$A"
+	then
+		after="$(expr x"$A" : 'x-A *\(.*\)$')"
+		q="$query&after=$after"
+	fi
+
+	nr=$limit
+	while test -n "$nr" && test $nr -eq $limit
+	do
+		# send curl and awk stdout to our stdout
+		# send curl stderr to our stderr
+		# capture awk stderr for eval
+		exec 3>&1
+		exec 4>&2
+		nr=
+		eval $(run_curl "$base$q" 2>&4 | awk -F'|' "$a" 2>&1 1>&3)
+		exec 2>&4
+		exec 1>&3
+		q="$query&after=$last"
+	done
+	;;
+cat)
+	parse_common '-r<RANGE>' '<KEY1> [<KEY2> ...]' "$@"
+	keys_to_urls
+	run_curl -L $r $urls
+	;;
+zcat)
+	parse_common '' '<KEY1> [<KEY2> ...]' "$@"
+	keys_to_urls
+
+	# favor pigz (http://zlib.net/pigz) as it's slightly faster
+	case "$(which pigz 2>/dev/null || :)" in
+	'') GZIP=gzip ;;
+	*/pigz) GZIP=pigz ;;
+	esac
+
+	run_curl -L $urls | $GZIP -dc
+	;;
+get)
+	parse_common '-r<RANGE>' '<KEY1> [<KEY2> ...]' "$@"
+	keys_to_urls -O
+	run_curl -L $r $urls
+	;;
+put)
+	parse_common '-f' '<KEY1> [<KEY2> ...]' "$@"
+	for file in $keys
+	do
+		test -f "$file" || die "E: FILE: $file is not a regular file"
+	done
+	for file in $keys
+	do
+		run_curl -H Expect: -T "$file" "$base/$file"
+	done
+	;;
+cp)
+	parse_common '-f' '<FILE> <KEY>' "$@"
+	test -n "$file" || die "E: <FILE> argument required"
+	test -n "$key" || die "E: <KEY> argument required"
+	test -f "$file" || die "E: FILE: $file is not a regular file"
+	run_curl -H Expect: -T "$file" "$base/$key"
+	;;
+tee)
+	parse_common '-f' '<KEY>' "$@"
+	test -n "$key" || die "E: <KEY> argument required"
+
+	# optimization: avoid writing to stdout if stdout is /dev/null
+	# this should work on Solaris, too, tested on Linux
+	is_null=$(ls -lL /dev/null /proc/$$/fd/1 2>/dev/null \
+	         | awk "$stdout_is_null")
+
+	if test -n "$is_null"
+	then
+		run_curl -H Expect: -T- "$base/$key"
+	else
+		teetmp="$(mktemp -t hyst.$command.$$.XXXXXXXX)"
+		rm $teetmp
+		trap 'rm -f $teetmp' EXIT
+		mkfifo $teetmp
+
+		run_curl -H Expect: -T- "$base/$key" < $teetmp >/dev/null &
+		# tee(1) feeds curl via fifo
+		tee $teetmp
+		wait
+	fi
+
+	;;
+stat)
+	parse_common '' '<KEY1> [<KEY2> ...]' "$@"
+	for i in $keys
+	do
+		test x"$i" = x"$key" || echo
+		run_curl -iI "$base/$i" | awk "BEGIN{key=\"$i\"}$stat_fmt"
+	done
+	;;
+rm)
+	parse_common '' '<KEY1> [<KEY2> ...]' "$@"
+	keys_to_urls
+	run_curl -X DELETE $urls
+	;;
+wait-repl)
+	parse_common '-t<TIMEOUT> -n<COUNT>' '<KEY>' "$@"
+	test -n "$key" || die "E: <KEY> argument required"
+	case $t in
+	'') timeout=3600 ;;
+	*)
+		timeout="$(expr x"$t" : 'x-t *\([0-9]\+\)$' || :)"
+		test -z "$timeout" && die "timeout=$t not an integer"
+		;;
+	esac
+	timeout=$(( $(date +%s) + $timeout ))
+	case $n in
+	'') count=2 ;;
+	*)
+		count="$(expr x"$n" : 'x-n *\([0-9]\+\)$' || :)"
+		test -z "$count" && die "count=$n not an integer"
+		;;
+	esac
+
+	while test $(date +%s) -lt $timeout
+	do
+		nr=$(run_curl -iI "$base/$key" | grep -ic '^X-Url-[0-9]\+:')
+		test $nr -ge $count && exit 0
+		sleep 1
+	done
+	exit 1
+	;;
+*) die "$usage" ;;
+esac