vines-services 0.1.0

data/lib/vines/services/storage/couchdb.rb

@@ -0,0 +1,157 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    class Storage
+      class CouchDB < Storage
+        register :couchdb
+
+        module ClassMethods
+          # Override CouchRest::Model::Persistence::ClassMethods#build_from_database
+          # to instantiate the fully qualified model class name. We store the
+          # bare class name in doc['type'] (e.g. Service rather than
+          # Vines::Services::CouchModels::Service), so outside query processes
+          # don't need to know our class hierarchy.
+          def build_from_database(doc = {}, options = {}, &block)
+            src = doc[model_type_key]
+            base = (src.blank? || src == self.to_s) ? self : "Vines::Services::CouchModels::#{src}".constantize
+            base.new(doc, options.merge(:directly_set_attributes => true), &block)
+          end
+
+          # CouchRest::Model uses Class#to_s to determine design document names
+          # as well as the value of doc['type']. Strip off all module names to
+          # get clean design document URLs.
+          def to_s
+            self.name.split('::').last
+          end
+        end
+
+        %w[host port database tls username password index_dir].each do |name|
+          define_method(name) do |*args|
+            if args.first
+              @config[name.to_sym] = args.first
+            else
+              @config[name.to_sym]
+            end
+          end
+        end
+
+        def initialize(&block)
+          @config = {}
+          instance_eval(&block)
+          [:host, :port, :database, :index_dir].each do |key|
+            raise "Must provide #{key}" unless @config[key]
+          end
+
+          @config[:index_dir] = File.expand_path(@config[:index_dir])
+          unless File.directory?(@config[:index_dir]) && File.writable?(@config[:index_dir])
+            raise 'Must provide a writable index directory'
+          end
+
+          @url = url(@config)
+          init_couch_rest
+
+          db = "%s-%s-%s.db" % [host, port, database]
+          @index = Indexer[File.join(@config[:index_dir], db)]
+        end
+
+        def get(path, &callback)
+          http(path, :get, &callback)
+        end
+
+        def post(path, body, &callback)
+          http(path, :post, body, &callback)
+        end
+
+        def delete(path, &callback)
+          http(path, :get) do |doc|
+            if doc
+              http("#{path}?rev=#{doc['_rev']}", :delete, &callback)
+            else
+              yield
+            end
+          end
+        end
+
+        def save(doc, &callback)
+          http('', :post, doc.to_json, &callback)
+        end
+
+        def index(system)
+          @index << system.ohai
+        end
+
+        def query(sql, *params, &callback)
+          @index.find(sql, params, &callback)
+        end
+
+        def store_file(path)
+          file = CouchModels::Upload.find_by_name(File.basename(path))
+          unless file
+            File.delete(path)
+            return
+          end
+
+          http = EM::HttpRequest.new("#{@url}/#{file.id}/data?rev=#{file.rev}").put(
+            head: {'Content-Type' => 'application/octet-stream'},
+            file: path
+          )
+          http.callback {|chunk| yield }
+        end
+
+        def create_views
+          # FIXME Use views in CouchRest::Model classes to populate db
+          designs = {}
+
+          EM.run do
+            http('', :put) do # create db
+              designs.each do |name, views|
+                get("/_design/#{name}") do |doc|
+                  doc ||= {"_id" => "_design/#{name}"}
+                  doc['language'] = 'javascript'
+                  doc['views'] = views
+                  save(doc) { EM.stop }
+                end
+              end
+            end
+          end
+        end
+
+        private
+
+        def http(path, method, body=nil)
+          args = {}.tap do |opts|
+            opts[:head] = {'Content-Type' => 'application/json'}
+            opts[:body] = body if body
+          end
+
+          http = EM::HttpRequest.new("#{@url}#{path}").send(method, args)
+          http.errback { yield }
+          http.callback do
+            doc = if (200...300).include?(http.response_header.status)
+              JSON.parse(http.response) rescue nil
+            end
+            yield doc
+          end
+        end
+
+        def url(config)
+          scheme = config[:tls] ? 'https' : 'http'
+          user, password = config.values_at(:username, :password)
+          credentials = empty?(user, password) ? '' : "%s:%s@" % [user, password]
+          "%s://%s%s:%s/%s" % [scheme, credentials, *config.values_at(:host, :port, :database)]
+        end
+
+        def init_couch_rest
+          *url, _ = @url.split('/')
+          server = CouchRest::Server.new(url.join('/'))
+          CouchRest::Model::Base.database = server.database(database)
+        end
+
+        def escape(jid)
+          URI.escape(jid, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/storage.rb

@@ -0,0 +1,33 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    class Storage
+      include Vines::Log
+
+      @@nicks = {}
+
+      # Register a nickname that can be used in the config file to specify this
+      # storage implementation.
+      def self.register(name)
+        @@nicks[name.to_sym] = self
+      end
+
+      def self.from_name(name, &block)
+        klass = @@nicks[name.to_sym]
+        raise "#{name} storage class not found" unless klass
+        klass.new(&block)
+      end
+
+      private
+
+      # Return true if any of the arguments are nil or empty strings.
+      # For example:
+      # username, password = 'alice@wonderland.lit', ''
+      # empty?(username, password) #=> true
+      def empty?(*args)
+        args.flatten.any? {|arg| (arg || '').strip.empty? }
+      end
+    end
+  end
+end

data/lib/vines/services/throttle.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    # Send many outgoing stanzas to the server at a sustained rate that won't
+    # cause the server to shutdown the component's stream with a policy_violation
+    # error.
+    class Throttle
+      def initialize(stream, delay=0.1)
+        @stream, @delay = stream, delay
+      end
+
+      # Send the nodes to the server at a constant rate. The nodes are sent
+      # asynchronously, so this method returns immediately.
+      def async_send(nodes)
+        timer = EM::PeriodicTimer.new(@delay) do
+          if node = nodes.shift
+            @stream.write(node)
+          else
+            timer.cancel
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/version.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    VERSION = '0.1.0'
+  end
+end

data/lib/vines/services/vql/compiler.rb

@@ -0,0 +1,94 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module VQL
+      # Compiles Vines Query Language queries into JavaScript for CouchDB views
+      # and SQL for sqlite queries. The output of the compiler is a fully formed
+      # query that can be sent directly to the database.
+      class Compiler
+        Citrus.load(File.expand_path('../vql.citrus', __FILE__))
+
+        # Return the compiled CouchDB map function. Raise an exception on
+        # compilation failure. The exception's error message can be shown to
+        # the user to debug the syntax.
+        def to_js(code)
+          raise ArgumentError, 'code required' if code.nil? || code.strip.empty?
+          expr = VinesQL.parse(code.strip)
+          %Q{
+            function(doc) {
+              if (doc.type != 'System') return;
+              try {
+                var match = #{expr.js};
+                if (match) {
+                  var name = doc['_id'].replace('system:', '');
+                  var os = doc.ohai.kernel.os.toLowerCase().replace('gnu/', '');
+                  emit(name, os);
+                }
+              } catch(e) {
+                log(e.message);
+              }
+            }
+          }
+        end
+
+        # Return the compiled CouchDB map function for all services. Raise an
+        # exception on compilation failure. The exception's error message can
+        # be shown to the user to debug the syntax.
+        def to_full_js(services)
+          maps = services.map do |service|
+            expr = VinesQL.parse(service.code.strip)
+            %Q{
+              try {
+                var match = #{expr.js};
+                if (match) {
+                  emit([0, '#{service.id}'], {name: name, os: os});
+                  emit([1, name], '#{service.id}');
+                }
+              } catch(e) {
+                log(e.message);
+              }
+            }
+          end
+
+          %Q{
+            function(doc) {
+              if (doc.type != 'System') return;
+              var name = doc['_id'].replace('system:', '');
+              var os = doc.ohai.kernel.os.toLowerCase().replace('gnu/', '');
+              #{maps.join}
+            }
+          }
+        end
+
+        # Return the compiled sqlite SQL query along with an array of parameter
+        # replacement values. Raise an exception on compilation failure. The
+        # exception's error message can be shown to the user to debug the syntax.
+        def to_sql(code)
+          raise ArgumentError, 'code required' if code.nil? || code.strip.empty?
+          expr = VinesQL.parse(code.strip)
+
+          keys, values = expr.params.partition.each_with_index {|p, ix| ix % 2 == 0 }
+          joins = keys.each_with_index.map do |k, ix|
+            "inner join attributes a%s on id=a%s.system_id and a%s.key=?" % [ix, ix, ix]
+          end
+
+          where = expr.sql.tap do |sql|
+            values.size.times do |ix|
+              sql.sub!(/(^|[^\.])value/, "\\1a#{ix}.value")
+            end
+          end
+
+          sql = %Q{
+            select name, os from systems
+            #{joins.join("\n")}
+            where #{where}
+            order by name
+          }
+
+          [sql, [keys, values].flatten]
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/vql/vql.citrus

@@ -0,0 +1,115 @@
+grammar VinesQL
+  rule expr
+    expr:(disjunctive | stmt)
+  end
+
+  rule disjunctive
+    (lhs:stmt 'or' ws rhs:expr) <Vines::Services::VQL::Or>
+  end
+
+  rule stmt
+    stmt:(conjunctive | term)
+  end
+
+  rule conjunctive
+    (lhs:term 'and' ws rhs:stmt) <Vines::Services::VQL::And>
+  end
+
+  rule term
+     is_not | is | ltgt | starts_with | ends_with | not_like | like | group
+  end
+
+  rule is
+    (lhs:member 'is' ws rhs:value) <Vines::Services::VQL::Is>
+  end
+
+  rule is_not
+    (lhs:member 'is' ws 'not' ws rhs:value) <Vines::Services::VQL::IsNot>
+  end
+
+  rule like
+    (lhs:member 'like' ws rhs:string) <Vines::Services::VQL::Like>
+  end
+
+  rule not_like
+    (lhs:member 'not' ws 'like' ws rhs:string) <Vines::Services::VQL::NotLike>
+  end
+
+  rule starts_with
+    (lhs:member 'starts' ws 'with' ws rhs:string) <Vines::Services::VQL::StartsWith>
+  end
+
+  rule ends_with
+    (lhs:member 'ends' ws 'with' ws rhs:string) <Vines::Services::VQL::EndsWith>
+  end
+
+  rule ltgt
+    (lhs:member op:([<>] '='?) ws rhs:number) <Vines::Services::VQL::LtGt>
+  end
+
+  rule value
+    string | number | keyword
+  end
+
+  rule group
+    (lparen expr rparen) <Vines::Services::VQL::Group>
+  end
+
+  rule member
+    (str:(ident ('.' ident)*) ws) <Vines::Services::VQL::Member>
+  end
+
+  rule ident
+    (alpha | '_') (alpha | '_' | digits)*
+  end
+
+  rule string
+    single_quoted | double_quoted
+  end
+
+  rule single_quoted
+    ("'" str:(~"'") "'" ws) <Vines::Services::VQL::SingleQuoted>
+  end
+
+  rule double_quoted
+    ('"' str:(~'"') '"' ws) <Vines::Services::VQL::DoubleQuoted>
+  end
+
+  rule keyword
+      (str:'null'  ws) <Vines::Services::VQL::Null>
+    | (str:'true'  ws) <Vines::Services::VQL::Terminal>
+    | (str:'false' ws) <Vines::Services::VQL::Terminal>
+  end
+
+  rule number
+    float | int
+  end
+
+  rule float
+    (str:('-'? digits '.' digits) ws) <Vines::Services::VQL::Terminal>
+  end
+
+  rule int
+    (str:('-'? digits) ws) <Vines::Services::VQL::Terminal>
+  end
+
+  rule digits
+    [0-9]+
+  end
+
+  rule alpha
+    [a-zA-Z]+
+  end
+
+  rule lparen
+    '(' ws
+  end
+
+  rule rparen
+    ')' ws
+  end
+
+  rule ws
+    [ \t\n\r]*
+  end
+end

data/lib/vines/services/vql/vql.rb

@@ -0,0 +1,186 @@
+module Vines
+  module Services
+    # These modules give semantic meaning to the Vines Query Language Citrus
+    # parser. The query language is translated into JavaScript and SQL fragments
+    # for use in CouchDB views or sqlite queries. This file must be loaded before
+    # vql.citrus.
+    module VQL
+      module Or
+        def js
+          "%s || %s" % [lhs.js, rhs.js]
+        end
+        def sql
+          "%s or %s" % [lhs.sql, rhs.sql]
+        end
+        def params
+          [lhs.params, rhs.params].flatten
+        end
+      end
+
+      module And
+        def js
+          "%s && %s" % [lhs.js, rhs.js]
+        end
+        def sql
+          "%s and %s" % [lhs.sql, rhs.sql]
+        end
+        def params
+          [lhs.params, rhs.params].flatten
+        end
+      end
+
+      module Is
+        def js
+          "(%s === %s)" % [lhs.js, rhs.js]
+        end
+        def sql
+          case rhs
+          when Vines::Services::VQL::Null
+            "value is null"
+          else
+            "value=?"
+          end
+        end
+        def params
+          case rhs
+          when Vines::Services::VQL::Null
+            [lhs.sql]
+          else
+            [lhs.sql, rhs.sql]
+          end
+        end
+      end
+
+      module IsNot
+        def js
+          "(%s !== %s)" % [lhs.js, rhs.js]
+        end
+        def sql
+          case rhs
+          when Vines::Services::VQL::Null
+            "value is not null"
+          else
+            "value <> ?"
+          end
+        end
+        def params
+          case rhs
+          when Vines::Services::VQL::Null
+            [lhs.sql]
+          else
+            [lhs.sql, rhs.sql]
+          end
+        end
+      end
+
+      module Like
+        def js
+          "(%s.indexOf(%s) !== -1)" % [lhs.js, rhs.js]
+        end
+        def sql
+          "value like ?"
+        end
+        def params
+          [lhs.sql, "%#{rhs.sql}%"]
+        end
+      end
+
+      module NotLike
+        def js
+          "(%s.indexOf(%s) === -1)" % [lhs.js, rhs.js]
+        end
+        def sql
+          "value not like ?"
+        end
+        def params
+          [lhs.sql, "%#{rhs.sql}%"]
+        end
+      end
+
+      module StartsWith
+        def js
+          "(%s.lastIndexOf(%s, 0) === 0)" % [lhs.js, rhs.js]
+        end
+        def sql
+          "value like ?"
+        end
+        def params
+          [lhs.sql, "#{rhs.sql}%"]
+        end
+      end
+
+      module EndsWith
+        def js
+          "(%s.match(%s + '$'))" % [lhs.js, rhs.js]
+        end
+        def sql
+          "value like ?"
+        end
+        def params
+          [lhs.sql, "%#{rhs.sql}"]
+        end
+      end
+
+      module LtGt
+        def js
+          "(%s %s %s)" % [lhs.js, op, rhs.js]
+        end
+        def sql
+          "cast(value as number) %s ?" % op
+        end
+        def params
+          [lhs.sql, rhs.sql]
+        end
+      end
+
+      module Group
+        def js
+          "(%s)" % expr.js
+        end
+        def sql
+          "(%s)" % expr.sql
+        end
+        def params
+          expr.params
+        end
+      end
+
+      module Terminal
+        def js
+          str.to_s
+        end
+        def sql
+          str.to_s
+        end
+        def params
+          []
+        end
+      end
+
+      module Member
+        include Terminal
+        def js
+          "doc.ohai.#{str}"
+        end
+      end
+
+      module SingleQuoted
+        include Terminal
+        def js
+          "'%s'" % str
+        end
+      end
+
+      module DoubleQuoted
+        include Terminal
+        def js
+          '"%s"' % str
+        end
+      end
+
+      module Null
+        include Terminal
+      end
+    end
+  end
+end

data/lib/vines/services.rb

@@ -0,0 +1,71 @@
+# encoding: UTF-8
+
+%w[
+  couchrest_model
+  logger
+  bcrypt
+  blather/client/client
+  cgi
+  citrus
+  em-http
+  fiber
+  fileutils
+  json
+  nokogiri
+  sqlite3
+  uri
+
+  vines/log
+  vines/daemon
+  vines/jid
+  vines/kit
+
+  vines/services/core_ext/blather
+  vines/services/core_ext/couchrest
+
+  vines/services/version
+  vines/services/connection
+  vines/services/config
+  vines/services/component
+  vines/services/priority_queue
+  vines/services/indexer
+  vines/services/throttle
+
+  vines/services/vql/vql
+  vines/services/vql/compiler
+
+  vines/services/command/init
+  vines/services/command/restart
+  vines/services/command/start
+  vines/services/command/stop
+  vines/services/command/views
+
+  vines/services/storage
+  vines/services/storage/couchdb
+  vines/services/storage/couchdb/fragment
+  vines/services/storage/couchdb/service
+  vines/services/storage/couchdb/system
+  vines/services/storage/couchdb/upload
+  vines/services/storage/couchdb/user
+  vines/services/storage/couchdb/vcard
+
+  vines/services/controller/base_controller
+  vines/services/controller/attributes_controller
+  vines/services/controller/disco_info_controller
+  vines/services/controller/uploads_controller
+  vines/services/controller/labels_controller
+  vines/services/controller/members_controller
+  vines/services/controller/messages_controller
+  vines/services/controller/probes_controller
+  vines/services/controller/services_controller
+  vines/services/controller/subscriptions_controller
+  vines/services/controller/systems_controller
+  vines/services/controller/transfers_controller
+  vines/services/controller/users_controller
+].each {|f| require f }
+
+module Vines
+  module Services
+    class Forbidden < StandardError; end
+  end
+end