reapack-index 1.0beta4 → 1.0rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b08f343ff91fe1f1f543b9ac82de72f698404ce8
-  data.tar.gz: 3b4f0c39955b88ba41f3bdf0759d7531d57e32ca
+  metadata.gz: 5cbcf419229c7231caca844f3b7f9e6800261b24
+  data.tar.gz: e3a8eabd17b759e7125f39276371383263130081
 SHA512:
-  metadata.gz: 545316c4c749f62d19f68b1f3cb82f447a5d8b0b7d9aaaaba815258ee86424e8f6c0e06cfa2c9ef8876b4eb52074f5a5ac193a954070602d859d2c078a4803cc
-  data.tar.gz: b0f6a34ed77612b71fe28a62cea6f0b6b0a7375506e8a2ecfc9ac90496172b2c30228a8212a064ae8c78e9d26103155ff493fab5d749b14ae4e04f2e4e17c5b7
+  metadata.gz: 7e04598f176942c766ade75dc7ec901793d3a58383e9e4e648d364f869d3798bdbc7b5239574ccc71b247bd115500c1c6c434de41353adc2611cbda46ed2bbf8
+  data.tar.gz: 42ebb6abbc5043e80b858d01ec64b94ec8c550e0a11a0c1daa964f3a7823b83a4282ef476d2c1e738b2242e781912d7cfe48bdcbba48bb943becb4bc3382a2f0

data/README.md

@@ -10,9 +10,10 @@ Subproject: [https://github.com/cfillion/metaheader](https://github.com/cfillion
 
 ### Installation
 
-Ruby 2 need to be installed on your computer and ready to be used from a command
-prompt. Run the following command from a command prompt (cmd.exe, Terminal.app,
-XTerm) to install reapack-index on your computer:
+Ruby v2.3 or newer must be installed on your computer in order to install
+and use this software.  
+Run the following command from a command prompt (eg. cmd.exe, Terminal.app,
+XTerm) to install reapack-index:
 
 ```
 gem install reapack-index
@@ -28,6 +29,7 @@ reapack-index [options] [path-to-your-repository]
 Modes:
     -c, --check                      Test every package including uncommited changes and exit
     -s, --scan [COMMIT]              Scan new commits (default) or specific commits
+        --rebuild                    Clear the index and rescan the whole git history
 Indexer options:
     -a, --[no-]amend                 Reindex existing versions
     -i, --ignore PATH                Don't check or index any file starting with PATH

data/lib/reapack/index.rb

@@ -12,6 +12,7 @@ require 'pandoc-ruby'
 require 'pathname'
 require 'rugged'
 require 'shellwords'
+require 'stable_sort'
 require 'time'
 
 require 'reapack/index/cdetector'
@@ -23,6 +24,7 @@ require 'reapack/index/named_node'
 require 'reapack/index/package'
 require 'reapack/index/parsers'
 require 'reapack/index/provides'
+require 'reapack/index/scanner'
 require 'reapack/index/source'
 require 'reapack/index/version'
 
@@ -36,31 +38,10 @@ class ReaPack::Index
     data: %w{data},
   }.freeze
 
-  WITH_MAIN = [:script, :effect].freeze
-
-  PROVIDES_VALIDATOR = proc {|value|
-    begin
-      Provides.parse_each(value).to_a and nil
-    rescue Error => e
-      e.message
-    end
-  }
-
-  HEADER_RULES = {
-    # package-wide tags
-    :version => [
-      MetaHeader::REQUIRED, MetaHeader::VALUE, MetaHeader::SINGLELINE, /\d/],
-
-    # version-specific tags
-    :author => [MetaHeader::VALUE, MetaHeader::SINGLELINE],
-    :changelog => [MetaHeader::VALUE],
-    :provides => [MetaHeader::VALUE, PROVIDES_VALIDATOR]
-  }.freeze
-
   FS_ROOT = File.expand_path('/').freeze
 
-  attr_reader :path, :url_template
-  attr_accessor :amend, :files, :time
+  attr_reader :path, :url_template, :cdetector
+  attr_accessor :amend, :commit, :files, :time
 
   class << self
     def is_type?(input)
@@ -125,11 +106,6 @@ class ReaPack::Index
     type = self.class.type_of path
     return unless type
 
-    # variables to restore if an error occur
-    backups = Hash[[:@doc, :@cdetector].map {|var|
-      [var, instance_variable_get(var).clone]
-    }]
-
     mh = MetaHeader.new contents
 
     if mh[:noindex]
@@ -137,42 +113,21 @@ class ReaPack::Index
       return
     end
 
-    if errors = mh.validate(HEADER_RULES)
-      raise Error, errors.join("\n")
-    end
+    # variables to restore if an error occur
+    backups = Hash[[:@doc, :@cdetector].map {|var|
+      [var, instance_variable_get(var).clone]
+    }]
 
     cat, pkg = package_for path
+    pkg.type = type
 
-    cselector = @cdetector[pkg.type = type, path]
-
-    pkg.version mh[:version] do |ver|
-      next unless ver.is_new? || @amend
+    scanner = Scanner.new cat, pkg, mh, self
 
-      # store the version name for make_url
-      @currentVersion = ver.name
-
-      ver.author = mh[:author]
-      ver.time = @time if @time && ver.is_new?
-      ver.changelog = mh[:changelog]
-
-      ver.replace_sources do
-        cselector.clear
-        sources = parse_provides mh[:provides], pkg
-
-        if WITH_MAIN.include?(type) && sources.none? {|src| src.file.nil? }
-          # add the package itself as a source
-          src = Source.new make_url(path)
-          sources.unshift src
-
-          cselector.push src.platform, path
-        end
-
-        sources.each {|src| ver.add_source src }
-      end
-    end
-
-    if cons = cselector.resolve
-      raise Error, cons.first
+    begin
+      scanner.run
+    rescue Error
+      backups.each {|var, value| instance_variable_set var, value }
+      raise
     end
 
     log_change 'new category', 'new categories' if cat.is_new?
@@ -190,16 +145,13 @@ class ReaPack::Index
     }
 
     bump_commit
-  rescue Error
-    backups.each {|var, value| instance_variable_set var, value }
-    raise
   end
-  
+
   def remove(path)
     cat, pkg = package_for path, false
     return unless pkg
 
-    @cdetector[pkg.type, path].clear
+    @cdetector[path].clear
 
     pkg.remove
     cat.remove if cat.empty?
@@ -253,7 +205,7 @@ class ReaPack::Index
       raise Addressable::URI::InvalidURIError
     end
 
-    @url_template = uri.to_s.freeze
+    @url_template = tpl
   rescue Addressable::URI::InvalidURIError
     raise Error, "invalid template '#{tpl}'"
   end
@@ -276,17 +228,15 @@ class ReaPack::Index
     log_change 'modified metadata' if oldName != newName
   end
 
-  def commit
-    @commit ||= @doc.root[:commit]
+  def last_commit
+    @doc.root[:commit]
   end
 
-  attr_writer :commit
-
   def write(path)
     sort @doc.root
 
     FileUtils.mkdir_p File.dirname(path)
-    File.write path, @doc.to_xml
+    File.binwrite path, @doc.to_xml
   end
 
   def write!
@@ -313,21 +263,9 @@ class ReaPack::Index
     list.join ', '
   end
 
-  def make_url(path, template = nil)
-    unless template
-      unless @url_template
-        raise Error, 'unable to generate download links: empty url template'
-      end
-
-      unless @files.include? path
-        raise Error, "file not found '#{path}'"
-      end
-    end
-
-    (template || @url_template)
-      .sub('$path', path)
-      .sub('$commit', commit || 'master')
-      .sub('$version', @currentVersion.to_s)
+  def clear
+    Category.find_all(@doc.root).each {|cat| cat.remove }
+    @cdetector.clear
   end
 
   def self.expand(filepath, basedir)
@@ -354,56 +292,19 @@ private
     [cat, pkg]
   end
 
-  def parse_provides(provides, pkg)
-    pathdir = Pathname.new pkg.category
-
-    Provides.parse_each(provides).map {|line|
-      line.file_pattern = pkg.name if line.file_pattern == '.'
-
-      expanded = self.class.expand line.file_pattern, pkg.category
-      cselector = @cdetector[line.type || pkg.type, pkg.path]
-
-      if expanded == pkg.path
-        # always resolve path even when an url template is set
-        files = [expanded]
-      elsif line.url_template.nil?
-        files = @files.select {|f| File.fnmatch expanded, f, File::FNM_PATHNAME }
-        raise Error, "file not found '#{line.file_pattern}'" if files.empty?
-      else
-        # use the relative path for external urls
-        files = [line.file_pattern]
-      end
-
-      files.map {|file|
-        src = Source.new make_url(file, line.url_template)
-        src.platform = line.platform
-        src.type = line.type
-
-        cselector.push src.platform, line.url_template ? expanded : file
-
-        if file != pkg.path
-          if line.url_template
-            src.file = file
-          else
-            src.file = Pathname.new(file).relative_path_from(pathdir).to_s
-          end
-        end
-
-        src
-      }
-    }.flatten
-  end
-
   def sort(node)
-    node.children.each {|n| sort n }
-
+    node.element_children.map {|n| sort n }
     return if node.name == Package.tag
-    sorted = node.children.sort_by{|n| n[:name].to_s }.sort_by {|n| n.name }
+
+    sorted = node.element_children
+      .stable_sort_by {|n| n[:name].to_s.downcase }
+      .stable_sort_by {|n| n.name.downcase }
+
     sorted.each {|n| node << n }
   end
 
   def bump_commit
-    sha1 = commit()
+    sha1 = @commit || last_commit
 
     if sha1.nil?
       @doc.root.remove_attribute 'commit'

data/lib/reapack/index/cdetector.rb

@@ -3,21 +3,26 @@ class ReaPack::Index
     Entry = Struct.new :key, :platform, :file
 
     class Selector
-      def initialize(bucket, key, cdetector)
-        @bucket, @key, @cdetector = bucket, key, cdetector
-        @entries = @cdetector.bucket bucket
+      def initialize(key, cdetector)
+        @key, @cdetector = key, cdetector
       end
 
-      def push(platform, file)
-        @entries << Entry.new(@key, platform, file).freeze
+      def push(bucket, platform, file)
+        @cdetector.bucket(bucket) << Entry.new(@key, platform, file).freeze
       end
 
       def clear
-        @entries.reject! {|e| e.key == @key }
+        @cdetector.buckets.each_value do |b|
+          b.reject! {|e| e.key == @key }
+        end
       end
 
       def resolve
-        @cdetector.resolve @bucket, @key
+        errors = @cdetector.buckets.map do |b, _|
+          @cdetector.resolve b, @key
+        end.compact.flatten
+
+        errors unless errors.empty?
       end
     end
 
@@ -31,28 +36,33 @@ class ReaPack::Index
         Hash[@buckets.map {|k, v| [k, v.clone] }]
     end
 
-    def [](bucket, key)
-      Selector.new bucket, key, self
+    def [](key)
+      Selector.new key, self
     end
 
+    attr_reader :buckets
+
     def bucket(name)
+      raise ArgumentError, 'bucket name is not a symbol' unless name.is_a? Symbol
       @buckets[name] ||= []
     end
 
-    def resolve(bucket, key = nil)
-      return unless bucket = @buckets[bucket]
+    def clear
+      @buckets.clear
+    end
 
-      dups = bucket.group_by {|e| e.file }.values.select {|a| a.size > 1 }
+    def resolve(bname, key = nil)
+      dups = bucket(bname).group_by {|e| e.file }.values.select {|a| a.size > 1 }
 
       errors = dups.map {|a|
         packages = a.map {|e| e.key }.uniq
         next if key && !packages.include?(key)
 
         if packages.size == 1 || !key
-          original = sort(a)[1]
+          original = sort_platforms(a)[1]
           msg = "duplicate file '#{original.file}'"
         else
-          original = sort(a.select {|e| e.key != key }).first
+          original = sort_platforms(a.select {|e| e.key != key }).first
           msg = "'#{original.file}' conflicts with '#{original.key}'"
         end
 
@@ -61,6 +71,7 @@ class ReaPack::Index
         if platforms.size > 1
           # check platform inheritance
           platforms.any? {|p|
+            break true unless Source::PLATFORMS[p]
             loop do
               p = Source::PLATFORMS[p] or break false
               break true if platforms.include? p
@@ -81,7 +92,9 @@ class ReaPack::Index
           pkgroot = File.join(cat.name, pkg.name)
 
           pkg.versions.last&.children(Source::TAG)&.each {|src|
-            entry = Entry.new pkgroot, src[:platform].to_sym
+            type = src[:type] || pkg.type
+            platform = src[:platform] || :all
+            entry = Entry.new pkgroot, platform.to_sym
 
             if src[:file]
               entry.file = ReaPack::Index.expand(src[:file], cat.name)
@@ -89,16 +102,17 @@ class ReaPack::Index
               entry.file = pkgroot
             end
 
-            bucket(pkg.type) << entry
+            bucket(type.to_sym) << entry
           }
         }
       }
     end
 
   private
-    def sort(set)
-      sorted = set.sort_by {|e| levels[e.platform] }
-      sorted.sort_by! {|e| Source::PLATFORMS.keys.index e.platform }
+    def sort_platforms(set)
+      set.group_by {|e| levels[e.platform] || 0 }.sort
+        .map {|_, a| a.sort_by {|e| Source::PLATFORMS.keys.index(e.platform) || 0 } }
+        .flatten
     end
 
     def levels

data/lib/reapack/index/cli.rb

@@ -77,10 +77,13 @@ private
       return
     end
 
-    if @opts[:scan].empty?
-      commits = @git.commits_since @index.commit.to_s
+    commits = if @opts[:rebuild]
+      @index.clear
+      @git.commits
+    elsif @opts[:scan].empty?
+      @git.commits_since @index.last_commit
     else
-      commits = @opts[:scan].map {|hash|
+      @opts[:scan].map {|hash|
         @git.get_commit hash or begin
           $stderr.puts '--scan: bad revision: %s' % @opts[:scan]
           @exit = false
@@ -105,7 +108,9 @@ private
     @index.time = commit.time
     @index.files = commit.filelist
 
-    commit.each_diff {|diff| process_diff diff }
+    commit.each_diff
+      .sort_by {|diff| diff.status == :deleted ? 0 : 1 }
+      .each {|diff| process_diff diff }
   ensure
     bump_progress
   end
@@ -190,18 +195,19 @@ private
   def do_check
     check_name
 
-    @index.amend = true # enable checks for released versions as well
+    @index.clear
     failures = []
 
     pkgs = Hash[Dir.glob("#{Regexp.quote(@git.path)}/**/*").sort.map {|abs|
       rel = @git.relative_path abs
       @index.files << rel
 
-      next if !File.file?(abs) || ignored?(abs) || !ReaPack::Index.is_package?(abs)
+      next if !File.file?(abs) || ignored?(abs) || !ReaPack::Index.is_package?(rel)
 
       [abs, rel]
     }.compact]
 
+    # reiterate over the pkg list after registering every file
     pkgs.each_pair {|abs, rel|
       begin
         @index.scan rel, File.read(abs)

data/lib/reapack/index/cli/options.rb

@@ -9,6 +9,7 @@ class ReaPack::Index::CLI
   DEFAULTS = {
     check: false,
     scan: [],
+    rebuild: false,
     verbose: false,
     warnings: true,
     progress: true,
@@ -53,7 +54,7 @@ class ReaPack::Index::CLI
       end
 
       op.on '-s', '--scan [COMMIT]', 'Scan new commits (default) or specific commits' do |commit|
-        opts[:check] = false
+        opts[:check] = opts[:rebuild] = false
         opts[:scan] ||= []
 
         if commit
@@ -63,6 +64,11 @@ class ReaPack::Index::CLI
         end
       end
 
+      op.on '--rebuild', 'Clear the index and rescan the whole git history' do
+        opts[:check] = false
+        opts[:rebuild] = true
+      end
+
       op.separator 'Indexer options:'
 
       op.on '-a', '--[no-]amend', 'Reindex existing versions' do |bool|