rere 0.0.1

data/example/solr/collection1/conf/xslt/updateXml.xsl

@@ -0,0 +1,70 @@
+<!-- 
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+<!--
+  Simple transform of Solr query response into Solr Update XML compliant XML.
+  When used in the xslt response writer you will get UpdaateXML as output.
+  But you can also store a query response XML to disk and feed this XML to
+  the XSLTUpdateRequestHandler to index the content. Provided as example only.
+  See http://wiki.apache.org/solr/XsltUpdateRequestHandler for more info
+ -->
+<xsl:stylesheet version='1.0' xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>
+  <xsl:output media-type="text/xml" method="xml" indent="yes"/>
+
+  <xsl:template match='/'>
+    <add>
+        <xsl:apply-templates select="response/result/doc"/>
+    </add>
+  </xsl:template>
+  
+  <!-- Ignore score (makes no sense to index) -->
+  <xsl:template match="doc/*[@name='score']" priority="100">
+  </xsl:template>
+
+  <xsl:template match="doc">
+    <xsl:variable name="pos" select="position()"/>
+    <doc>
+        <xsl:apply-templates>
+          <xsl:with-param name="pos"><xsl:value-of select="$pos"/></xsl:with-param>
+        </xsl:apply-templates>
+    </doc>
+  </xsl:template>
+
+  <!-- Flatten arrays to duplicate field lines -->
+  <xsl:template match="doc/arr" priority="100">
+      <xsl:variable name="fn" select="@name"/>
+      
+      <xsl:for-each select="*">
+		<xsl:element name="field">
+		    <xsl:attribute name="name"><xsl:value-of select="$fn"/></xsl:attribute>
+	        <xsl:value-of select="."/>
+		</xsl:element>
+      </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template match="doc/*">
+      <xsl:variable name="fn" select="@name"/>
+
+	<xsl:element name="field">
+	    <xsl:attribute name="name"><xsl:value-of select="$fn"/></xsl:attribute>
+        <xsl:value-of select="."/>
+	</xsl:element>
+  </xsl:template>
+
+  <xsl:template match="*"/>
+</xsl:stylesheet>

data/example/solr/solr.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+   This is an example of a simple "solr.xml" file for configuring one or 
+   more Solr Cores, as well as allowing Cores to be added, removed, and 
+   reloaded via HTTP requests.
+
+   More information about options available in this configuration file, 
+   and Solr Core administration can be found online:
+   http://wiki.apache.org/solr/CoreAdmin
+-->
+
+<!--
+ All (relative) paths are relative to the Solr Home Directory
+  
+  persistent: Save changes made via the API to this file
+  sharedLib: path to a lib directory that will be shared across all cores
+-->
+<solr persistent="true">
+	<!-- by default, this is 50 @ WARN
+  <logging enabled="true">
+  	<watcher size="100" threshold="INFO" />
+  </logging>
+   -->
+
+  <!--
+  adminPath: RequestHandler path to manage cores.  
+    If 'null' (or absent), cores will not be manageable via request handler
+  defaultCoreName: (optional) core to use when no core name is specified in an access url
+    
+  All of the attributes in cores after defaultCoreName only apply when running in SolrCloud mode.
+  You can read more about SolrCloud mode at http://wiki.apache.org/solr/SolrCloud
+  -->
+  <cores adminPath="/admin/cores" defaultCoreName="collection1" host="${host:}" hostPort="${jetty.port:8983}" hostContext="${hostContext:solr}" zkClientTimeout="${zkClientTimeout:15000}">
+    <core name="collection1" instanceDir="collection1" />
+  </cores>
+</solr>

data/example/solr/zoo.cfg

@@ -0,0 +1,17 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+
+# the directory where the snapshot is stored.
+# dataDir=/opt/zookeeper/data
+# NOTE: Solr defaults the dataDir to <solrHome>/zoo_data
+
+# the port at which the clients will connect
+# clientPort=2181
+# NOTE: Solr sets this based on zkRun / zkHost params
+

data/lib/rere.rb

@@ -0,0 +1,66 @@
+require 'net/http'
+require 'rere/version'
+require 'rere/server'
+
+if defined?(Rails)
+  require 'rere/railtie'
+end
+
+module Rere
+  extend self
+
+  def start(timeout = 10)
+    server.start
+
+    # TODO: Maybe add this to the server.start method itself
+    wait_for_startup(timeout)
+  end
+
+  def run
+    server.run
+  end
+
+  def stop
+    server.stop
+    puts "*** Stopped solr ***"
+  end
+
+  def reload_core(corename)
+    server.reload_core(corename)
+  end
+
+  def log_path
+    server.log_path
+  end
+
+
+  def server
+    @server ||= begin
+                  env = ENV['RAILS_ENV'] || 'development'
+                  Rere::Server.new(:environment => env)
+                end
+  end
+
+
+  # Hmmm, should I explicitly ping the cores too?
+  def running?
+    begin
+      ping_uri = URI.parse("#{server.url}/admin/ping")
+      Net::HTTP.get(ping_uri)
+      true
+    rescue
+      false
+    end
+  end
+
+  private
+
+  def wait_for_startup(timeout)
+    timeout.times do
+      break if running?
+      sleep(1)
+    end
+    raise TimeoutError, "Solr didn't start within #{timeout}" unless running?
+  end
+
+end

data/lib/rere/railtie.rb

@@ -0,0 +1,8 @@
+module Rere
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load "rere/tasks/solr.rake"
+    end
+  end
+end
+

data/lib/rere/server.rb

@@ -0,0 +1,380 @@
+require 'fileutils'
+require 'tempfile'
+require 'escape'
+require 'uri'
+require 'yaml'
+require 'childprocess'
+
+class Hash
+  def symbolize_keys!
+    keys.each do |key|
+      self[(key.to_sym rescue key) || key] = delete(key)
+    end
+    self
+  end
+end
+
+module Rere
+
+  #
+  # Wraps the basic operations for starting/stopping Solr in the Moto
+  # environment.
+  #
+  # By default when starting the server will look for a config/solr.yml
+  # file. If found these options will be used as the default options for
+  # solr. Any arguments provided on the command line will override those
+  # in the solr.yml file.
+  #
+  # = Configuration options
+  #
+  # <tt>environment</tt> - (required) the environment solr is starting in. The
+  # server will look up the config options under the given environment in
+  # config/solr.yml.
+  #
+  # <tt>config_path</tt> - use to specify a config file path for the server
+  # if it is not in the default location of config/solr.yml
+  #
+  class Server
+
+    ServerError         = Class.new(RuntimeError)
+    ConfigError         = Class.new(ServerError)
+    AlreadyRunningError = Class.new(ServerError)
+    NotRunningError     = Class.new(ServerError)
+    JavaMissing         = Class.new(ServerError)
+
+
+    # Jetty opts
+    attr_reader :server_root, :solr_home, :solr_data_dir, :tmp_dir, :port, :bind_address
+
+    # Java opts
+    attr_reader :min_memory, :max_memory, :log_level
+
+    # Environment opts
+    attr_reader :environment, :install_root, :config_path
+
+    # Solr opts
+    attr_reader :log_dir
+
+    # TODO: maybe explicitly read the options and check for anything that is
+    # not understood
+    def initialize(opts = {})
+      @environment = opts.delete(:environment) {|k| raise(ArgumentError, ":environment is required") }
+      @install_root = opts.fetch(:install_root) { File.join(Dir.pwd) }
+
+      # load the config for the server
+      @config_path = opts.delete(:config_file) {|k|  "#{install_root}/config/solr.yml" }
+      unless File.exist?(@config_path)
+        raise ConfigError, "No configuration found. Looking for #{@config_path}"
+      end
+      puts "Loading config from #{@config_path} and environment #{@environment}"
+     
+      config = YAML::load(File.open(@config_path))[@environment]['solr'] 
+      config.symbolize_keys!
+
+      config = config.merge(opts)
+      @min_memory = config.fetch(:min_memory) { 1024 }
+      @max_memory = config.fetch(:max_memory) { 1024 }
+      if config[:url]
+        @bind_address, @port = parse_url(config[:url])
+      else
+        @bind_address = config.delete(:hostname) {|k| raise(ArgumentError, ":hostname is required") }
+        @port = config[:port] || 8983
+      end
+
+      @log_level = config[:log_level] || 'INFO'
+      @log_dir = log_dir_path(config[:log_dir])
+      
+      @solr_data_dir = config.delete(:data_dir) { "#{install_root}/solr/data/#{environment}" }
+
+      @solr_home = config.delete(:solr_home) { "#{install_root}/solr" }
+
+    end
+
+    def log_dir_path(log_dir = nil)
+      if log_dir.nil?
+        "#{install_root}/log" 
+      else
+        File.expand_path(log_dir)
+      end
+    end
+
+    def url
+      @url ||= begin
+        if bind_address == '0.0.0.0'
+          "http://localhost:#{port}/solr"
+        else
+          "http://#{bind_address}:#{port}/solr"
+        end
+      end
+    end
+
+    def parse_url(url)
+      uri = URI.parse(url)
+      return uri.host, uri.port
+    end
+
+    # Esssentially sends a message like:
+    #
+    # http://localhost:8983/solr/admin/cores?action=RELOAD&core=core0
+    #
+    def reload_core(corename)
+      reload_url = "#{url}/admin/cores?action=RELOAD&core=#{corename}"
+      response = Net::HTTP.get_response(URI.parse(reload_url))
+      if response.code.to_i == 200
+        puts "Reloaded #{corename} core"
+      else
+        raise "Could not reload #{corename}: response #{response.code}. " +
+          "Check your solr log: #{log_path}"
+      end
+    end
+
+    # There is a lag between the time solr starts and it is ready to
+    # accept requests. Use +running?+ to determine if the process itself
+    # is up. 
+    def running?
+      if File.exists?(pid_path)
+        existing_pid = IO.read(pid_path).to_i
+        begin
+          Process.kill(0, existing_pid)
+          return true
+        rescue Errno::ESRCH
+          return false
+        end
+      else
+        return false
+      end
+    end
+
+    def status
+      if running?
+        ctime = File.stat(pid_path).ctime
+        pid = IO.read(pid_path).to_i
+        puts "Running since #{ctime} with pid #{pid}"
+      else
+        puts "stopped"
+      end
+    end
+
+    def start
+      # TODO make sure java is installed?
+
+      ensure_dirs!
+
+      if File.exists?(pid_path)
+        existing_pid = IO.read(pid_path).to_i
+        begin
+          Process.kill(0, existing_pid)
+          raise(AlreadyRunningError, "Server is already running with PID #{existing_pid}")
+        rescue Errno::ESRCH
+          $stderr.puts("Removing stale PID #{pid_path}")
+          FileUtils.rm(pid_path)
+        end
+
+      end
+
+      @logging_config_path = write_logging_properties
+
+      # fork the start script, which execs the call to java (thus replaces itself)
+      fork do
+        pid = fork do
+          Process.setsid
+          $stdin.reopen('/dev/null')
+          $stdout.reopen('/dev/null', 'a')
+          $stderr.reopen($stdout)
+          run
+        end
+        write_pid(pid)
+
+        puts '*'*80
+        puts "Solr starting in environment #{environment}, using config #{config_path}"
+        puts "and solr home #{solr_home}."
+        puts "Server will be available at 'http://#{bind_address}:#{port}/solr'" 
+        puts "With process id #{pid} and pid file #{pid_path}"
+        puts "Logging to #{log_path}"
+        puts '*'*80
+      end
+
+    end
+
+    def stop(raise_if_not_running = true)
+      if File.exist?(pid_path)
+        pid = IO.read(pid_path).to_i
+        puts "Stopping solr with process #{pid} and pid #{pid_path}"
+
+        begin
+          Process.kill('TERM', pid)
+        rescue Errno::ESRCH
+          raise NotRunningError, "Process with PID #{pid} is not running!"
+        ensure
+          FileUtils.rm(pid_path)
+        end
+      else
+        # raise NotRunningError, "Solr is not running. Looking for pid file #{pid_path}"
+        if raise_if_not_running
+          raise NotRunningError, "Solr is not running. Looking for pid file #{pid_path}"
+        else
+          puts "Solr not running. Not pid found at #{pid_path}"
+        end
+
+      end
+    end
+
+    def restart
+      stop(false)
+      sleep(2)
+      start
+    end
+
+    def run
+      puts '*'*80
+      puts "Solr starting in environment #{environment}, using config #{config_path}"
+      puts "and solr home #{solr_home}."
+      puts "Server will be available at 'http://#{bind_address}:#{port}/solr'" 
+      puts '*'*80
+      
+      cmd = ["java"]
+      cmd << "-server" unless windows?
+      cmd << "-Xms#{min_memory}m"
+      cmd << "-Xmx#{max_memory}m"
+      cmd << "-Djetty.port=#{port}"
+      cmd << "-Drails.env=#{environment}" # Only needed while we connect to the db for synonyms
+      cmd << "-Dsolr.solr.home=#{solr_home}"
+      cmd << "-Dsolr.data.dir=#{solr_data_dir}"
+      cmd << "-Djava.io.tmpdir=#{tmp_dir}"
+      cmd << "-DRAILS_ENV=#{environment}"
+      
+      if @logging_config_path
+        puts "Loading logging configuration from #{@logging_config_path}"
+        cmd << "-Dlog4j.configuration=file://#{@logging_config_path}" 
+      end
+
+      cmd << "-jar"
+      cmd << "start.jar"
+
+      cmd_s = Escape.shell_command(cmd)
+
+puts "RUNNING: #{cmd_s}"
+      
+      # This doesn't work with jruby
+      # FileUtils.cd(server_root) do
+      #   exec("#{cmd_s}")
+      # end
+
+      FileUtils.cd(File.dirname(solr_jar)) do
+        require "childprocess"
+        process = ChildProcess.build(*cmd)
+
+        trap('TERM') do
+          puts "Stopping solr"
+          process.stop
+          exit 0
+        end
+
+        trap('INT') do 
+          puts "Stopping solr"
+          process.stop
+          exit 0
+        end
+
+        process.io.inherit!
+        process.start
+        process.wait
+      end
+
+    end
+
+    def solr_jar
+      "#{server_root}/start.jar"
+    end
+
+    def ensure_dirs!
+      FileUtils.mkdir_p pid_dir
+    end
+
+    def pid_dir
+      @pid_dir || File.join(tmp_dir, 'pids')
+    end
+
+    def pid_file
+      @pid_file || "solr.#{environment}.pid"
+    end
+
+    def pid_path
+      File.join(pid_dir, pid_file)
+    end
+
+    def write_pid(pid)
+      File.open(pid_path, 'w') {|f| f << pid }
+    end
+
+    def environment
+      @environment || 'development'
+    end
+
+    def tmp_dir
+      File.expand_path(@tmp_dir || "#{install_root}/tmp")
+    end
+
+    def gem_root
+      @gem_root ||= File.expand_path("../../../", __FILE__)
+    end
+
+    # The directory containing jetty and the solr.war. 
+    def server_root
+      @server_root || File.expand_path('../../../server', __FILE__)
+    end
+
+    # Directory where the index directories: index and spelling
+    # are kept. 
+    # Defaults to "#{install_root}/solr/data/#{environment}"
+    def solr_data_dir
+      @solr_data_dir 
+    end
+    alias_method :data_dir, :solr_data_dir
+
+    def solr_home
+      @solr_home # || "#{install_root}/solr"
+    end
+
+    def log_path
+      @log_path || "#{log_dir}/solr.#{environment}.log"
+    end
+
+    # Write out a java properties files with the necessary 
+    # key value pairs to initialize log4j correctly. 
+    # Returns the path to where the properties file is created
+    def write_logging_properties
+      logging_config_path = File.join(tmp_dir, "solr.#{environment}.log.properties")
+      File.open(logging_config_path, 'w') {|f| f.puts logging_config }
+      logging_config_path
+    end
+
+    private
+
+    def logging_config
+      <<-LOG_CONFIG
+log4j.rootLogger=#{log_level.to_s.upcase}, file, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x \u2013 %m%n
+
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.MaxFileSize=4MB
+log4j.appender.file.MaxBackupIndex=9
+
+log4j.appender.file.File=#{log_path}
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m%n
+
+log4j.logger.org.apache.zookeeper=WARN
+LOG_CONFIG
+    end
+
+    def windows?
+      RUBY_PLATFORM =~ /mswin32/ || RUBY_PLATFORM =~ /mingw/
+    end
+    
+  end
+
+end