gemstash 1.0.0.pre.1

data/lib/gemstash/cli/start.rb

@@ -0,0 +1,52 @@
+require "gemstash"
+require "puma/cli"
+
+module Gemstash
+  class CLI
+    # This implements the command line start task to start the Gemstash server:
+    #  $ gemstash start
+    class Start < Gemstash::CLI::Base
+      def run
+        prepare
+        setup_logging
+        store_daemonized
+        Puma::CLI.new(args, Gemstash::Logging::StreamLogger.puma_events).run
+      end
+
+    private
+
+      def setup_logging
+        return unless daemonize?
+        Gemstash::Logging.setup_logger(gemstash_env.base_file("server.log"))
+      end
+
+      def store_daemonized
+        Gemstash::Env.daemonized = daemonize?
+      end
+
+      def daemonize?
+        @cli.options[:daemonize]
+      end
+
+      def puma_config
+        File.expand_path("../../puma.rb", __FILE__)
+      end
+
+      def args
+        config_args + pidfile_args + daemonize_args
+      end
+
+      def config_args
+        ["--config", puma_config]
+      end
+
+      def daemonize_args
+        if daemonize?
+          ["--daemon"]
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/cli/status.rb

@@ -0,0 +1,21 @@
+require "gemstash"
+require "puma/control_cli"
+
+module Gemstash
+  class CLI
+    # This implements the command line status task to check the server status:
+    #  $ gemstash status
+    class Status < Gemstash::CLI::Base
+      def run
+        prepare
+        Puma::ControlCLI.new(args).run
+      end
+
+    private
+
+      def args
+        pidfile_args + %w(status)
+      end
+    end
+  end
+end

data/lib/gemstash/cli/stop.rb

@@ -0,0 +1,21 @@
+require "gemstash"
+require "puma/control_cli"
+
+module Gemstash
+  class CLI
+    # This implements the command line stop task to stop the Gemstash server:
+    #  $ gemstash stop
+    class Stop < Gemstash::CLI::Base
+      def run
+        prepare
+        Puma::ControlCLI.new(args).run
+      end
+
+    private
+
+      def args
+        pidfile_args + %w(stop)
+      end
+    end
+  end
+end

data/lib/gemstash/config.ru

@@ -0,0 +1,13 @@
+require "gemstash"
+require "puma/commonlogger"
+
+use Rack::Deflater
+use Gemstash::Logging::RackMiddleware
+
+if Gemstash::Env.daemonized?
+  use Puma::CommonLogger, Gemstash::Logging::StreamLogger.for_stdout
+end
+
+use Gemstash::Env::RackMiddleware, Gemstash::Env.current
+use Gemstash::GemSource::RackMiddleware
+run Gemstash::Web.new(gemstash_env: Gemstash::Env.current)

data/lib/gemstash/configuration.rb

@@ -0,0 +1,41 @@
+require "yaml"
+
+module Gemstash
+  #:nodoc:
+  class Configuration
+    DEFAULTS = {
+      :cache_type => "memory",
+      :base_path => File.expand_path("~/.gemstash"),
+      :db_adapter => "sqlite3",
+      :bind => "tcp://0.0.0.0:9292",
+      :rubygems_url => "https://www.rubygems.org"
+    }.freeze
+
+    DEFAULT_FILE = File.expand_path("~/.gemstash/config.yml").freeze
+
+    def initialize(file: nil, config: nil)
+      if config
+        @config = DEFAULTS.merge(config).freeze
+        return
+      end
+
+      file ||= DEFAULT_FILE
+
+      if File.exist?(file)
+        @config = YAML.load_file(file)
+        @config = DEFAULTS.merge(@config)
+        @config.freeze
+      else
+        @config = DEFAULTS
+      end
+    end
+
+    def default?(key)
+      @config[key] == DEFAULTS[key]
+    end
+
+    def [](key)
+      @config[key]
+    end
+  end
+end

data/lib/gemstash/db.rb

@@ -0,0 +1,15 @@
+require "gemstash"
+
+module Gemstash
+  # Module containing the DB models.
+  module DB
+    raise "Gemstash::DB cannot be loaded until the Gemstash::Env is available" unless Gemstash::Env.available?
+    Sequel::Model.db = Gemstash::Env.current.db
+    Sequel::Model.raise_on_save_failure = true
+    Sequel::Model.plugin :timestamps, update_on_create: true
+    autoload :Authorization, "gemstash/db/authorization"
+    autoload :Dependency,    "gemstash/db/dependency"
+    autoload :Rubygem,       "gemstash/db/rubygem"
+    autoload :Version,       "gemstash/db/version"
+  end
+end

data/lib/gemstash/db/authorization.rb

@@ -0,0 +1,20 @@
+require "gemstash"
+
+module Gemstash
+  module DB
+    # Sequel model for authorizations table.
+    class Authorization < Sequel::Model
+      def self.insert_or_update(auth_key, permissions)
+        db.transaction do
+          record = self[auth_key: auth_key]
+
+          if record
+            record.update(permissions: permissions)
+          else
+            create(auth_key: auth_key, permissions: permissions)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/db/dependency.rb

@@ -0,0 +1,50 @@
+require "gemstash"
+
+module Gemstash
+  module DB
+    # Sequel model for dependencies table.
+    class Dependency < Sequel::Model
+      def self.insert_by_spec(version_id, spec)
+        spec.runtime_dependencies.each do |dep|
+          requirements = dep.requirement.requirements
+          requirements = requirements.map {|r| "#{r.first} #{r.last}" }
+          requirements = requirements.join(", ")
+          create(version_id: version_id,
+                 rubygem_name: dep.name,
+                 requirements: requirements)
+        end
+      end
+
+      def self.fetch(gems)
+        results = db["
+          SELECT rubygem.name,
+                 version.number, version.platform,
+                 dependency.rubygem_name, dependency.requirements
+          FROM rubygems rubygem
+          JOIN versions version
+            ON version.rubygem_id = rubygem.id
+          LEFT JOIN dependencies dependency
+            ON dependency.version_id = version.id
+          WHERE rubygem.name IN ?
+            AND version.indexed = ?", gems.to_a, true].to_a
+        results.group_by {|r| r[:name] }.each do |gem, rows|
+          requirements = rows.group_by {|r| [r[:number], r[:platform]] }
+
+          value = requirements.map do |version, r|
+            deps = r.map {|x| [x[:rubygem_name], x[:requirements]] }
+            deps = [] if deps.size == 1 && deps.first.first.nil?
+
+            {
+              :name => gem,
+              :number => version.first,
+              :platform => version.last,
+              :dependencies => deps
+            }
+          end
+
+          yield(gem, value)
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/db/rubygem.rb

@@ -0,0 +1,14 @@
+require "gemstash"
+
+module Gemstash
+  module DB
+    # Sequel model for rubygems table.
+    class Rubygem < Sequel::Model
+      def self.find_or_insert(spec)
+        record = self[name: spec.name]
+        return record.id if record
+        new(name: spec.name).tap(&:save).id
+      end
+    end
+  end
+end

data/lib/gemstash/db/version.rb

@@ -0,0 +1,51 @@
+require "gemstash"
+
+module Gemstash
+  module DB
+    # Sequel model for versions table.
+    class Version < Sequel::Model
+      many_to_one :rubygem
+
+      def deindex
+        update(indexed: false)
+      end
+
+      def reindex
+        update(indexed: true)
+      end
+
+      # This converts to the format used by /private/specs.4.8.gz
+      def to_spec
+        [rubygem.name, Gem::Version.new(number), platform]
+      end
+
+      def self.for_spec_collection(prerelease: false)
+        where(indexed: true, prerelease: prerelease).association_join(:rubygem)
+      end
+
+      def self.find_by_spec(gem_id, spec)
+        self[rubygem_id: gem_id,
+             number: spec.version.to_s,
+             platform: spec.platform.to_s]
+      end
+
+      def self.find_by_full_name(full_name)
+        result = self[full_name: full_name]
+        return result if result
+        # Try again with the default platform, in case it is implied
+        self[full_name: "#{full_name}-ruby"]
+      end
+
+      def self.insert_by_spec(gem_id, spec)
+        gem_name = Gemstash::DB::Rubygem[gem_id].name
+        new(rubygem_id: gem_id,
+            number: spec.version.to_s,
+            platform: spec.platform.to_s,
+            full_name: "#{gem_name}-#{spec.version}-#{spec.platform}",
+            storage_id: spec.full_name,
+            indexed: true,
+            prerelease: spec.version.prerelease?).tap(&:save).id
+      end
+    end
+  end
+end

data/lib/gemstash/dependencies.rb

@@ -0,0 +1,93 @@
+require "cgi"
+require "set"
+
+module Gemstash
+  #:nodoc:
+  class Dependencies
+    def self.for_private
+      new(scope: "private", db_model: Gemstash::DB::Dependency)
+    end
+
+    def self.for_upstream(upstream, http_client)
+      new(scope: "upstream/#{upstream}", http_client: http_client)
+    end
+
+    def initialize(scope: nil, http_client: nil, db_model: nil)
+      @scope = scope
+      @http_client = http_client
+      @db_model = db_model
+    end
+
+    def fetch(gems)
+      Fetcher.new(gems, @scope, @http_client, @db_model).fetch
+    end
+
+    #:nodoc:
+    class Fetcher
+      include Gemstash::Env::Helper
+      include Gemstash::Logging
+
+      def initialize(gems, scope, http_client, db_model)
+        @gems = Set.new(gems)
+        @scope = scope
+        @http_client = http_client
+        @db_model = db_model
+        @dependencies = []
+      end
+
+      def fetch
+        fetch_from_cache
+        fetch_from_database
+        fetch_from_web
+        cache_missing
+        @dependencies
+      end
+
+    private
+
+      def done?
+        @gems.empty?
+      end
+
+      def fetch_from_cache
+        gemstash_env.cache.dependencies(@scope, @gems) do |gem, value|
+          @gems.delete(gem)
+          @dependencies += value
+        end
+      end
+
+      def fetch_from_database
+        return if done?
+        return unless @db_model
+        log.info "Querying dependencies: #{@gems.to_a.join(", ")}"
+
+        @db_model.fetch(@gems) do |gem, value|
+          @gems.delete(gem)
+          gemstash_env.cache.set_dependency(@scope, gem, value)
+          @dependencies += value
+        end
+      end
+
+      def fetch_from_web
+        return if done?
+        return unless @http_client
+        log.info "Fetching dependencies: #{@gems.to_a.join(", ")}"
+        gems_param = @gems.map {|gem| CGI.escape(gem) }.join(",")
+        fetched = @http_client.get("api/v1/dependencies?gems=#{gems_param}")
+        fetched = Marshal.load(fetched).group_by {|r| r[:name] }
+
+        fetched.each do |gem, result|
+          @gems.delete(gem)
+          gemstash_env.cache.set_dependency(@scope, gem, result)
+          @dependencies += result
+        end
+      end
+
+      def cache_missing
+        @gems.each do |gem|
+          gemstash_env.cache.set_dependency(@scope, gem, [])
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/env.rb

@@ -0,0 +1,150 @@
+require "gemstash"
+require "dalli"
+require "fileutils"
+require "sequel"
+require "uri"
+
+module Gemstash
+  # Storage for application-wide variables and configuration.
+  class Env
+    # The Gemstash::Env must be set before being retreived via
+    # Gemstash::Env.current. This error is thrown when that is not honored.
+    class EnvNotSetError < StandardError
+    end
+
+    # Little module to provide easy access to the current Gemstash::Env.
+    module Helper
+    private
+
+      def gemstash_env
+        Gemstash::Env.current
+      end
+    end
+
+    # Rack middleware to set the Gemstash::Env for the app.
+    class RackMiddleware
+      def initialize(app, gemstash_env)
+        @app = app
+        @gemstash_env = gemstash_env
+      end
+
+      def call(env)
+        env["gemstash.env"] = @gemstash_env
+        @app.call(env)
+      end
+    end
+
+    def initialize(config = nil, cache: nil, db: nil)
+      @config = config
+      @cache = cache
+      @db = db
+    end
+
+    def self.available?
+      !Thread.current[:gemstash_env].nil?
+    end
+
+    def self.current
+      raise EnvNotSetError unless Thread.current[:gemstash_env]
+      Thread.current[:gemstash_env]
+    end
+
+    def self.current=(value)
+      Thread.current[:gemstash_env] = value
+    end
+
+    def self.daemonized?
+      if @daemonized.nil?
+        raise "Daemonized hasn't been set yet!"
+      else
+        @daemonized
+      end
+    end
+
+    def self.daemonized=(value)
+      value = false if value.nil?
+      @daemonized = value
+    end
+
+    def config
+      @config ||= Gemstash::Configuration.new
+    end
+
+    def config=(value)
+      reset
+      @config = value
+    end
+
+    def reset
+      @config = nil
+      @cache = nil
+      @cache_client = nil
+      @db = nil
+    end
+
+    def base_path
+      dir = config[:base_path]
+
+      if config.default?(:base_path)
+        FileUtils.mkpath(dir) unless Dir.exist?(dir)
+      else
+        raise "Base path '#{dir}' is not writable" unless File.writable?(dir)
+      end
+
+      dir
+    end
+
+    def base_file(path)
+      File.join(base_path, path)
+    end
+
+    def rackup
+      File.expand_path("../config.ru", __FILE__)
+    end
+
+    def db
+      @db ||= begin
+        case config[:db_adapter]
+        when "sqlite3"
+          db_path = base_file("gemstash.db")
+
+          if RUBY_PLATFORM == "java"
+            db = Sequel.connect("jdbc:sqlite:#{db_path}", max_connections: 1)
+          else
+            db = Sequel.connect("sqlite://#{URI.escape(db_path)}", max_connections: 1)
+          end
+        when "postgres"
+          db = Sequel.connect(config[:db_url])
+        else
+          raise "Unsupported DB adapter: '#{config[:db_adapter]}'"
+        end
+
+        Gemstash::Env.migrate(db)
+        db
+      end
+    end
+
+    def self.migrate(db)
+      Sequel.extension :migration
+      migrations_dir = File.expand_path("../migrations", __FILE__)
+      Sequel::Migrator.run(db, migrations_dir, :use_transactions => true)
+    end
+
+    def cache
+      @cache ||= Gemstash::Cache.new(cache_client)
+    end
+
+    def cache_client
+      @cache_client ||= begin
+        case config[:cache_type]
+        when "memory"
+          Gemstash::LruReduxClient.new
+        when "memcached"
+          Dalli::Client.new(config[:memcached_servers])
+        else
+          raise "Invalid cache client: '#{config[:cache_type]}'"
+        end
+      end
+    end
+  end
+end