webvac 0.1.2

data/doc/TODO

@@ -0,0 +1,15 @@
+Unordered:
+
+· The closure abuse in Serv precludes using the URL to generate the routes.
+  This needs a fix in order to generalize beyond Pleroma.
+· Should be easy to stream rather than loading everything into memory,
+  but until then, big-ish files (≈2MB) take a second to get out of venti.
+  Obviously, it'll be faster and more reliable to implement the venti protocol.
+· Stats and webvac-unsweep.  This will allow hot objects to be swapped out of
+  venti.
+· Implement the venti protocol instead of calling $plan9bin/vac.
+· Pool venti servers.
+· Redis does a fine job for the lookup tables, but I cannot shake the feeling
+  that I should be using something else.  Maybe I'm just overly nervous about
+  it.
+· Etags?

data/doc/nginx.example.conf

@@ -0,0 +1,31 @@
+pid		/tmp/nginx.pid;
+error_log	/tmp/nginx-error.log info;
+daemon off;
+http {
+	include	/etc/nginx/mime.types;
+	access_log  /tmp/nginx.log;
+	server {
+		listen   8890;
+		server_name  localhost;
+
+		# These two location blocks are the interesting part of the config file:
+		location /media {
+			root /media/www;
+			try_files $uri @media;
+		}
+		location @media {
+			proxy_pass http://localhost:8891;
+			proxy_set_header Host $http_host;
+		}
+	}
+
+	# I have no idea why *all* of this is required:
+	client_body_temp_path	/tmp;
+	proxy_temp_path	/tmp;
+	fastcgi_temp_path	/tmp;
+	uwsgi_temp_path	/tmp;
+	scgi_temp_path	/tmp;
+}
+events {
+	worker_connections	1024;
+}

data/lib/webvac.rb

@@ -0,0 +1,159 @@
+%w(
+	redic
+	magic
+	json
+	time
+	cgi
+).each &method(:require)
+
+# The namespace for WebVac.  See the README.
+module WebVac
+	# Config object, intended to be used as a singleton.
+	class Config
+		# The default config options.  See the README.
+		Defaults = {
+			redis_url: "redis://localhost:6379/0",
+
+			server_path_strip: "/media",
+			server_path_prepend: "/media/block/fse",
+
+			venti_server: 'localhost',
+
+			plan9bin: '/opt/plan9/bin',
+
+			mime_substitutions: {
+				'text/html' => 'text/plain',
+			},
+		}
+		attr_accessor *Defaults.keys
+
+		# The sorted list of places where we will look for config files
+		# to load.
+		ConfigPaths = [
+			ENV['WEBVAC_CONFIG'],
+			"./config/webvac.json",
+			"#{ENV['HOME']}/.webvac.json",
+			"/etc/webvac.json",
+		].compact
+
+		# Reads/parses config and instantiates an object
+		def self.load
+			f = ConfigPaths.find { |f| File.readable?(f) }
+			cfg = if f
+				JSON.parse File.read(f)
+			else
+				{}
+			end
+			new cfg
+		end
+
+		# Takes a config, replaces the defaults with it.
+		# Will throw exceptions if you give it a bad config, you should probably
+		# just call Config.load.
+		def initialize cfg
+			Defaults.each { |k,v|
+				send("#{k}=", v)
+			}
+			cfg.each { |k,v|
+				send("#{k}=", v)
+			}
+		end
+
+		def path_fixup path
+			@_path_rx ||= /^#{Regexp.escape(server_path_strip)}/
+			path.sub(@_path_rx, server_path_prepend)
+		end
+	end
+
+	# Stateless-ish client for venti.
+	# I completely punted on implementing a venti client, so it just calls
+	# the vac/unvac binaries.  Does the job!
+	class Vac
+		attr_reader :config
+
+		# Takes an instance of Config.
+		def initialize cfg
+			@config = cfg
+		end
+
+		def save! fn
+			contents = File.read(fn)
+			pi, po = IO.pipe
+			io = IO.popen(
+				{'venti' => config.venti_server},
+				["#{config.plan9bin}/vac", '-i', File.basename(fn)],
+				in: pi
+			).tap { |io| Thread.new { Process.wait(io.pid) } }
+			po.write contents
+			po.close
+			io.read.chomp.sub(/^vac:/, '')
+		end
+
+		def load! vac
+			unless /^vac:[a-f0-9]{40}$/.match(vac)
+				raise ArgumentError, "#{vac.inspect} not a vac score?"
+			end
+			IO.popen(
+				{'venti' => config.venti_server},
+				["#{config.plan9bin}/unvac", '-c', vac]
+			).tap { |io| Thread.new { Process.wait(io.pid) } }.read
+		end
+	end
+
+	# Sits in front of Redis (just Redis right now), and handles the mapping
+	# of vac hashes to pathnames, as well as the metadata (in JSON and in the
+	# form of HTTP headers, which allows HEAD requests to be cheap).  Also does
+	# some of the bookkeeping necessary for that, like the interaction with
+	# libmagic.
+	#
+	# Relatively threadsafe, but maintains one Redis connection per active
+	# thread (created on demand).
+	class Table
+		attr_reader :config
+
+		# Takes an instance of Config.
+		def initialize cfg
+			@config = cfg
+		end
+
+		# Takes a filename, returns the filename's metadata.  Stateless-ish.
+		def fn2md f
+			s = File.stat(f)
+			m = {
+				'Content-Type' => Magic.guess_file_mime_type(f),
+				'Content-Length' => s.size.to_s,
+				'Last-Modified' => s.mtime.rfc822,
+			} rescue nil
+		end
+
+		def meta_save! fn, sc
+			md = fn2md(fn)
+			return unless md
+			redis.call 'HSET', 'score2md', sc, md.to_json
+		end
+
+		def metadata score
+			# Overall, doesn't really matter if this fails.
+			JSON.parse(
+				redis.call('HGET', 'score2md', score.sub(/^vac:/, ''))
+			) rescue nil
+		end
+
+		def rec_score! fn, sc
+			redis.call 'HSET', 'path2score', fn, sc
+		end
+
+		def redis
+			Thread.current[:webvac_redis] ||= Redic.new(config.redis_url)
+		end
+
+		def path2score p
+			r = redis.call 'HGET', 'path2score', p
+			return "vac:#{r}" if r
+		end
+
+		def guess_mime contents
+			Magic.guess_string_mime_type(contents)
+		end
+	end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: webvac
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Pete
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: watts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: pete@debu.gs
+executables:
+- webvac-sweep
+- webvac-server
+extensions: []
+extra_rdoc_files:
+- doc/LICENSE
+- doc/TODO
+- doc/nginx.example.conf
+- README
+files:
+- README
+- Rakefile
+- bin/webvac-server
+- bin/webvac-sweep
+- conf/rainbows.rb
+- config.ru
+- doc/LICENSE
+- doc/TODO
+- doc/nginx.example.conf
+- lib/webvac.rb
+homepage: http://github.com/pete/webvac
+licenses:
+- AGPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: UGC management/backup using venti
+test_files: []