buildr 0.14.0

data/lib/core/transports.rb

@@ -0,0 +1,381 @@
+require "net/ssh"
+require "net/sftp"
+require "uri/sftp"
+require "digest/md5"
+require "digest/sha1"
+require "facet/progressbar"
+require "highline"
+
+
+# Monkeypatching: SFTP never defines the mkdir method on its session or the underlying
+# driver, it just redirect calls through method_missing. Rake, on the other hand, decides
+# to define mkdir on Object, and so routes our calls to FileUtils.
+class Net::SFTP::Session
+  def mkdir(path, attrs = {})
+    method_missing :mkdir, path, attrs
+  end
+end
+
+class Net::SFTP::Protocol::Driver
+  def mkdir(first, path, attrs = {})
+    method_missing :mkdir, first, path, attrs
+  end
+end
+
+
+module Buildr
+  module Transports
+
+    class NotFound < Exception
+    end
+
+    # Perform one or more operations using an open connection to the
+    # specified URL. For examples, see Transport#download and Transport#upload.
+    def self.perform(url, options = nil, &block)
+      uri = URI.parse(url.to_s)
+      const_get(uri.scheme.upcase).perform(uri, options, &block)
+    end
+
+    # Convenience method for downloading a single file from the specified
+    # URL to the target file.
+    def self.download(url, target, options = nil)
+      uri = URI.parse(url.to_s)
+      path, uri.path = uri.path, ""
+      const_get(uri.scheme.upcase).perform(uri, options) do |transport|
+        transport.download(path, target)
+      end
+    end
+
+    class Transport
+
+      class << self
+
+        # Perform one or more operations using an open connection to the
+        # specified URL. For examples, see #download and #upload.
+        def perform(url, options = nil)
+          instance = new(url, options)
+          begin
+            yield instance
+          ensure
+            instance.close
+          end
+        end
+      end
+
+      # The server URI.
+      attr_reader :uri
+      # The base path on the server, always ending with a slash.
+      attr_reader :base_path
+      # Options passed during construction.
+      attr_reader :options
+
+      def initialize(url, options)
+        @uri = URI.parse(url.to_s)
+        @base_path = @uri.path || "/"
+        @base_path += "/" unless @base_path[-1] == ?/
+        @options = options || {}
+      end
+
+      # Downloads a file from the specified path, relative to the
+      # server URI. Downloads to either the target file, or by
+      # calling the block with each chunk of the file.
+      #
+      # For example:
+      #   Transports.perform("http://server/libs") do |http|
+      #     http.download("my_project/test.jar", "test.jar")
+      #     http.download("my_project/readme") { |text| $stdout.write text }
+      #   end
+      def download(path, target, &block)
+        fail "Upload not implemented for this transport"
+      end
+
+      # Uploads a file (source) to the server at the specified path,
+      # relative to the server URI.
+      #
+      # For example:
+      #   Transports.perform("sftp://server/libs") do |sftp|
+      #     sftp.mkpath "my_project"
+      #     sftp.upload("target/test.jar", "my_project/test.jar")
+      #   end
+      def upload(source, path)
+        fail "Upload not implemented for this transport"
+      end
+
+      # Creates a path on the server relative to the server URI.
+      # See #upload for example.
+      def mkpath(path)
+        fail "Upload not implemented for this transport"
+      end
+
+    protected
+
+      # :call-seq:
+      #   with_progress_bar(file_name, length) { |progress| ... }
+      #
+      # Displays a progress bar while executing the block. The first
+      # argument provides a filename to display, the second argument
+      # its size in bytes.
+      #
+      # The block is yielded with a progress object that implements
+      # a single method. Call << for each block of bytes down/uploaded.
+      def with_progress_bar(file_name, length)
+        if verbose && $stdout.isatty
+          progress_bar = Console::ProgressBar.new(file_name, length)
+          # Extend the progress bar so we can display count/total.
+          class << progress_bar
+            def total()
+              convert_bytes(@total)
+            end
+          end
+          # Squeeze the filename into 30 characters.
+          if file_name.size > 30
+            base, ext = file_name.split(".")
+            truncated = "#{base[0..26-ext.size]}...#{ext}"
+          else
+            truncated = file_name
+          end
+          progress_bar.format = "#{truncated}: %3d%% %s %s/%s %s"
+          progress_bar.format = "%3d%% %s %s/%s %s"
+          progress_bar.format_arguments = [:percentage, :bar, :bytes, :total, :stat] 
+          progress_bar.bar_mark = "."
+
+       
+          begin
+            class << progress_bar
+              def <<(bytes)
+                inc bytes.respond_to?(:size) ? bytes.size : bytes
+              end
+            end
+            yield progress_bar
+          ensure
+            progress_bar.finish
+          end
+        else
+          progress_bar = Object.new
+          class << progress_bar
+            def <<(bytes)
+            end
+          end
+          yield progress_bar
+        end
+      end
+
+      # :call-seq:
+      #   with_digests(types?) { |digester| ... } => hash
+      #
+      # Use the Digester to create digests for files you are downloading or
+      # uploading, and either verify their signatures (download) or create
+      # signatures (upload).
+      #
+      # The method takes one argument with the list of digest algorithms to
+      # support. Leave if empty and it will default to MD5 and SHA1.
+      #
+      # The method then yields the block passing it a Digester. The Digester
+      # supports two methods. Use << to pass data that you are down/uploading.
+      # Once all data is transmitted, use each to iterate over the digests.
+      # The each method calls the block with the digest type (e.g. "md5")
+      # and the hexadecimal digest value.
+      #
+      # For example:
+      #   with_digests do |digester|
+      #     download url do |block|
+      #       digester << block
+      #     end
+      #     digester.each do |type, hexdigest|
+      #       signature = download "#{url}.#{type}"
+      #       fail "Mismatch" unless signature == hexdigest
+      #     end
+      #   end
+      def with_digests(types = nil)
+        digester = Digester.new(types)
+        yield digester
+        digester.to_hash
+      end
+
+      class Digester
+
+        def initialize(types)
+          types ||= [ "md5", "sha1" ]
+          @digests = types.inject({}) do |hash, type|
+            hash[type.to_s.downcase] = Digest.const_get(type.to_s.upcase).new
+            hash
+          end
+        end
+
+        # Add bytes for digestion.
+        def <<(bytes)
+          @digests.each { |type, digest| digest << bytes }
+        end
+
+        # Iterate over all the digests calling the block with two arguments:
+        # the digest type (e.g. "md5") and the hexadecimal digest value. 
+        def each()
+          @digests.each { |type, digest| yield type, digest.hexdigest }
+        end
+
+        # Returns a hash that maps each digest type to its hexadecimal digest value.
+        def to_hash()
+          @digests.keys.inject({}) do |hash, type|
+            hash[type] = @digests[type].hexdigest
+            hash
+          end
+        end
+
+      end
+
+    end
+
+
+    class HTTP < Transport
+
+      def initialize(url, options)
+        super
+        @http = Net::HTTP.start(@uri.host, @uri.port)
+      end
+
+      def download(path, target = nil, &block)
+        puts "Requesting #{path} from #{@uri}" if Rake.application.options.trace
+        @http.request_get(@base_path + path) do |response|
+          case response
+          when Net::HTTPRedirection
+            # Try to download from the new URI, handle relative redirects.
+            puts "Redirected" if Rake.application.options.trace
+            Transports.download(@uri + URI.parse(response["location"]), target, @options)
+
+          when Net::HTTPOK
+            puts "Downloading #{@uri}/#{path}" if verbose
+            with_progress_bar path.split("/").last, response.content_length do |progress|
+              with_digests(@options[:digests]) do |digester|
+
+                download = proc do |write|
+                  # Read the body of the page and write it out.
+                  response.read_body do |chunk|
+                    write[chunk]
+                    digester << chunk
+                    progress << chunk 
+                  end
+                  # Check server digests before approving the download.
+                  digester.each do |type, hexdigest|
+                    @http.request_get("#{@base_path}#{path}.#{type.to_s.downcase}") do |response|
+                      if Net::HTTPOK === response
+                        puts "Checking signature from #{@uri}/#{path}" if Rake.application.options.trace
+                        fail "Checksum failure for #{@uri}/#{path}: #{type.to_s.upcase} digest on server did not match downloaded file" unless
+                          response.read_body.split.first == hexdigest
+                      end
+                    end
+                  end
+                end
+            
+                if target
+                  # If download breaks we end up with a partial file which is
+                  # worse than not having a file at all, so download to temporary
+                  # file and then move over.
+                  temp = Tempfile.new(File.basename(target))
+                  download[ proc { |chunk| temp.write chunk } ]
+                  temp.close
+                  File.move temp.path, target if target
+                else
+                  download[ block ]
+                end
+
+              end
+            end
+          when Net::HTTPNotFound
+            raise NotFound
+          else
+            fail "Failed to download #{@uri}/#{path}: #{response.message}"
+          end
+        end
+      end
+
+      def close()
+        @http.finish
+        @http = nil
+      end
+
+    end
+
+
+    class SFTP < Transport
+
+      class << self
+        def passwords()
+          @passwords ||= {}
+        end
+      end
+
+      attr_reader :sftp
+
+      def initialize(url, options)
+        super
+        # SSH options are based on the username/password from the URI.
+        ssh_options = { :port=>@uri.port, :username=>@uri.user }.merge(options || {})
+        ssh_options[:password] ||= SFTP.passwords[@uri.host]
+        begin
+          puts "Connecting to #{@uri.host}" if Rake.application.options.trace
+          session = Net::SSH.start(@uri.host, ssh_options)
+          SFTP.passwords[@uri.host] = ssh_options[:password]
+        rescue Net::SSH::AuthenticationFailed=>ex
+          # Only if running with console, prompt for password.
+          if !ssh_options[:password] && $stdout.isatty
+            password = HighLine.new.ask("Password for #{@uri.host}:") { |q| q.echo = "*" }
+            ssh_options[:password] = password
+            retry
+          end
+          raise
+        end
+        @sftp = session.sftp.connect
+        puts "connected" if Rake.application.options.trace
+      end
+
+      def upload(source, path)
+        File.open(source) do |file|
+          with_progress_bar path.split("/").last, File.size(source) do |progress|
+            with_digests(@options[:digests]) do |digester|
+              puts "Uploading to #{@base_path}#{path}" if Rake.application.options.trace
+              @sftp.open_handle(@base_path + path, "w") do |handle|
+                # Writing in chunks gives us the benefit of a progress bar,
+                # but also require that we maintain a position in the file,
+                # since write() with two arguments always writes at position 0.
+                pos = 0
+                while chunk = file.read(32 * 4096) 
+                  @sftp.write(handle, chunk, pos)
+                  pos += chunk.size
+                  digester << chunk
+                  progress << chunk
+                end
+              end
+
+              # Upload all the digests.
+              digester.each do |type, hexdigest|
+                puts "Uploading signature to #{@base_path}#{path}.#{type}" if Rake.application.options.trace
+                @sftp.open_handle("#{@base_path}#{path}.#{type}", "w") do |handle|
+                  @sftp.write(handle, "#{hexdigest}  #{path}")
+                end
+              end
+            end
+
+          end 
+        end
+      end
+
+      def mkpath(path)
+        # To create a path, we need to create all its parent.
+        # We use realpath to determine if the path already exists,
+        # otherwise mkdir fails.
+        puts "Creating path #{@base_path}" if Rake.application.options.trace
+        path.split("/").inject(@base_path) do |base, part|
+          @sftp.realpath(base+part) rescue @sftp.mkdir base + part
+          "#{base}#{part}/"
+        end
+      end
+
+      def close()
+        @sftp.close
+        @sftp = nil
+      end
+
+    end
+
+  end
+end

data/lib/java/compile.rb

@@ -0,0 +1,316 @@
+module Buildr
+  module Java
+
+    class CompileTask < Rake::Task
+
+      # Compiler options, accessible from Compiler#options.
+      #
+      # Supported options are:
+      # - warnings -- Generate warnings if true (opposite of -nowarn).
+      # - verbose -- Output messages about what the compiler is doing.
+      # - deprecation -- Output source locations where deprecated APIs
+      #   are used.
+      # - source -- Source compatibility with specified release.
+      # - target -- Class file compatibility with specified release.
+      # - lint -- Value to pass to xlint argument. Use true to enable
+      #   default lint options, or pass a specific setting as string
+      #   or array of strings.
+      # - debug -- Generate debugging info.
+      # - other -- Array of options to pass to the Java compiler as is.
+      #
+      # For example:
+      #   options.verbose = true
+      #   options.source = options.target = "1.6"
+      class Options
+
+        include Attributes
+
+        OPTIONS = [:warnings, :verbose, :deprecation, :source, :target, :lint, :debug, :other]
+
+        # Generate warnings (opposite of -nowarn).
+        inherited_attr :warnings 
+        # Output messages about what the compiler is doing.
+        inherited_attr :verbose
+        # Output source locations where deprecated APIs are used.
+        inherited_attr :deprecation
+        # Provide source compatibility with specified release.
+        inherited_attr :source
+        # Generate class files for specific VM version.
+        inherited_attr :target
+        # Values to pass to Xlint: string or array. Use true to enable
+        # Xlint with no values.
+        inherited_attr :lint
+        # Generate all debugging info.
+        inherited_attr :debug
+        # Array of arguments passed to the Java compiler as is.
+        inherited_attr :other
+
+        def initialize(parent = nil)
+          @parent = parent
+        end
+
+        attr_reader :parent 
+
+        def clear()
+          OPTIONS.each { |name| send "#{name}=", nil }
+        end
+
+        def to_s()
+          OPTIONS.inject({}){ |hash, name| hash[name] = send(name) ; hash }.reject{ |name,value| value.nil? }.inspect
+        end
+
+        # Returns Javac command line arguments from the set of options.
+        def javac_args()
+          args = []  
+          args << "-nowarn" unless warnings && Rake.application.options.trace
+          args << "-verbose" if verbose
+          args << "-g" if debug
+          args << "-deprecation" if deprecation
+          args << ["-source", source.to_s] if source
+          args << ["-target", target.to_s] if target
+          case lint
+          when Array
+            args << "-Xlint:#{lint.join(',')}"
+          when String
+            args << "-Xlint:#{lint}"
+          when true
+            args << "-Xlint"
+          end
+          args << other if other
+          args
+        end
+
+      end
+
+      # The target directory for the generated class files.
+      attr_accessor :target
+
+      def initialize(*args)
+        super
+        if name[":"] # Only if in namespace
+          parent = Rake::Task["^compile"]
+          if parent && parent.respond_to?(:options)
+            @options = Options.new(parent.options)
+          end
+        end
+
+        enhance do |task|
+          # Do we have any files to compile?
+          if target && files.empty?
+            puts "All source files are up to date" if Rake.application.options.trace && !sources.empty?
+          elsif target
+            mkpath target, :verbose=>false
+            Java.javac files, :sourcepath=>sourcepath, :classpath=>classpath,
+              :output=>target, :javac_args=>options.javac_args, :name=>task.name
+            # By touching the target we let other tasks know we did something,
+            # and also prevent recompiling again for classpath dependencies.
+            touch target, :verbose=>false
+          end
+        end
+      end
+
+      # An array of source directories and files.
+      def sources()
+        @sources ||= []
+      end
+
+      def sources=(paths)
+        @sources = paths.flatten
+      end
+
+      # Array of classpath dependencies: files, file tasks, artifacts specs.
+      def classpath()
+        @classpath ||= []
+      end
+
+      def classpath=(paths)
+        @classpath = paths.flatten
+      end
+
+      # Returns the compiler options.
+      def options()
+        @options ||= Options.new
+      end
+
+      # Sets the compile options based on a hash of values, or reset to
+      # the default by passing nil.
+      def options=(arg)
+        case arg
+        when Options
+          @options = arg
+        when Hash
+          options.clear
+          arg.each { |k,v| options.send "#{k}=", v }
+        when nil
+          options.clear
+        else
+          raise ArgumentError, "Expecting Options, hash or nil (to reset)"
+        end
+        @options
+      end
+
+      # Sets the target directory and returns self. For example:
+      #   compile(src_dir).to(target_dir).with(artifacts)
+      def to(dir)
+        self.target = dir
+        self
+      end
+
+      # Adds files and artifacts to the classpath and returns self.
+      # For example:
+      #   compile(src_dir).to(target_dir).with(artifact, file, task)
+      def with(*args)
+        self.classpath |= args.flatten
+        self
+      end
+
+      # Sets the compiler options and returns self. For example:
+      #   compile(src_dir).using(:warnings=>true, :verbose=>true)
+      def using(hash)
+        self.options = hash
+        self
+      end
+
+      # Returns true if any classes were compiled.
+      def compiled?()
+        @files && !@files.empty?
+      end
+
+    protected
+
+      # Returns the real classpath. Uses the values of #classpath, but resolves
+      # artifact specifications, projects and other conveniences, executes tasks
+      # (see #sanitize) and returns a compact array of unique file names.
+      def real_classpath()
+        @real_classpath ||= sanitize(artifacts(classpath))
+      end
+
+      # Return the sourcepath, essentialy compact array of directory and file names,
+      # as set by the user, but after invoking dependencies (see #sanitize).
+      def sourcepath()
+        @real_sources ||= sanitize(sources)
+        @real_sources.select { |source| File.directory?(source) }
+      end
+
+      # Returns the files to compile. This list is derived from the list of sources,
+      # expanding directories into files, and includes only source files that are
+      # newer than the corresponding class file. Includes all files if one or more
+      # classpath dependency has been updated.
+      def files()
+        unless @files
+          @real_sources ||= sanitize(sources)
+          # Compile all files if we compiled nothing, or one of the classpath
+          # dependencies is newer than the compiled classes.
+          all = !File.exist?(target) ||
+            File.stat(target).mtime < (real_classpath.collect{ |file| File.stat(file).mtime } + [ Rake::EARLY ]).max
+
+          if all # Do not restrict to changed files
+            @files = @real_sources.collect do |source|
+              File.directory?(source) ? Dir[File.join(source, "**", "*.java")] : source
+            end.flatten
+          else
+            # Only changed files.
+            @files = @real_sources.collect do |source|
+              if File.directory?(source)
+                Dir[File.join(source, "**", "*.java")].select do |java|
+                  klass = java.sub(source, target).ext(".class")
+                  !File.exist?(klass) || File.stat(java).mtime > File.stat(klass).mtime
+                end
+              else
+                source
+              end
+            end.flatten
+          end
+        end
+        @files
+      end
+
+      def sanitize(paths)
+        # Flatten to handle nested arrays often used with dependencies.
+        # Invoke all tasks, treated as prerequisites (e.g. download artifacts).
+        # Return task name or file name, ignoring nils and duplicates.
+        paths.flatten.each { |path| path.invoke if path.respond_to?(:invoke) }.
+          collect { |path| path.respond_to?(:name) ? path.name : path.to_s }.
+          compact.uniq
+      end
+
+    end
+
+  end
+
+
+  # Create and return a compiler task. The task is name "compile" in the current
+  # namespace. Method arguments are passed as sources to compile, and options are
+  # inherited from any compile task in a parent namespace.
+  #
+  # For example:
+  #   compile("src").to("classes").with(artifact, file, task)
+  def self.compile(*sources)
+    returning(Java::CompileTask.define_task("compile")) do |task|
+      task.sources |= sources
+    end
+  end
+
+  def compile(*sources)
+    Buildr.compile(*sources)
+  end
+
+  # Global task compiles all projects.
+  desc "Compile all projects"
+  LocalDirectoryTask.define_task "compile"
+
+
+  class Project
+
+    # The source directory. The default is "src".
+    inherited_attr :src_dir, "src"
+    # The Java source code directory. The default is <src_dir>/main/java.
+    inherited_attr :java_src_dir do File.join(src_dir, "main", "java") end
+    # The resources source directory. The default is <src_dir>/main/resources.
+    inherited_attr :resources_dir do File.join(src_dir, "main", "resources") end
+    # The target directory. The default is "target".
+    inherited_attr :target_dir, "target"
+    # The Java target directory. The default is <target_dir>/classes.
+    inherited_attr :java_target_dir do File.join(target_dir, "classes") end
+
+    def resources(*tasks, &block)
+      returning(@resources_task ||= Filter.define_task("resources")) do |task|
+        task.enhance tasks, &block
+      end
+    end
+
+    def compile(*sources, &block)
+      returning(@compile_task ||= Java::CompileTask.define_task("compile")) do |task|
+        task.sources |= sources
+        task.enhance &block if block
+      end
+    end
+
+    def prepare(*tasks, &block)
+      returning(@prepare_task ||= task("prepare")) do |task|
+        task.enhance tasks, &block
+      end
+    end
+
+  end
+
+
+  Project.on_create do |project|
+    # Prepare is prerequisite for compile, resources follows compile.
+    project.compile.enhance([project.prepare]) { |task| project.resources.invoke }
+    project.recursive_task("compile")
+    task("build").enhance [ project.compile ]
+    task("clean") { rm_rf project.path_to(:target_dir), :verbose=>false }
+
+    project.enhance do |project|
+      # Automagic compilation only if the source directory exists.
+      project.compile.sources << project.path_to(:java_src_dir) if File.exist?(project.path_to(:java_src_dir))
+      project.compile.target ||= project.path_to(:java_target_dir)
+      project.resources.include project.path_to(:resources_dir, "*") if File.exist?(project.path_to(:resources_dir))
+      project.resources.target ||= project.compile.target
+      file(project.compile.target).enhance([ project.compile, project.resources ])
+    end
+
+  end
+
+end