reapack-index 1.0beta3 → 1.0beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a8fcd949ead46a7e862d70e2d45f886aa13a48e5
-  data.tar.gz: b44bf961705b9582abc694e5a4782171002b0bcb
+  metadata.gz: b08f343ff91fe1f1f543b9ac82de72f698404ce8
+  data.tar.gz: 3b4f0c39955b88ba41f3bdf0759d7531d57e32ca
 SHA512:
-  metadata.gz: 0204c31e2c928ccb7dda6f430701bccc6952473efbf855853e5b86d8c9f916cddb9871953871841efe75302d76d2c5ea92a75835cd7e4e2731186e3b05664566
-  data.tar.gz: 9c39f26aef7f0249d6a7a78a7d58edc8c4ce9bc1078db134872b06cd955967eddba9d452fd70fa358e6d5897c3d022fe14ed66b007fb2f2138e15b4455c5a829
+  metadata.gz: 545316c4c749f62d19f68b1f3cb82f447a5d8b0b7d9aaaaba815258ee86424e8f6c0e06cfa2c9ef8876b4eb52074f5a5ac193a954070602d859d2c078a4803cc
+  data.tar.gz: b0f6a34ed77612b71fe28a62cea6f0b6b0a7375506e8a2ecfc9ac90496172b2c30228a8212a064ae8c78e9d26103155ff493fab5d749b14ae4e04f2e4e17c5b7

data/.travis.yml

@@ -1,6 +1,6 @@
 language: ruby
 rvm:
-  - 2.2.3
+  - 2.3.0
 before_install:
   - gem update bundler
   - sudo apt-get -qq update

data/README.md

@@ -1,4 +1,4 @@
-# Package indexer for ReaPack-based repositories
+# Package indexer for git-based ReaPack repositories
 
 Parent project: [https://github.com/cfillion/reapack](https://github.com/cfillion/reapack)  
 Subproject: [https://github.com/cfillion/metaheader](https://github.com/cfillion/metaheader)
@@ -25,19 +25,23 @@ reapack-index [options] [path-to-your-repository]
 ```
 
 ```
-Options:
-    -a, --[no-]amend                 Reindex existing versions
+Modes:
     -c, --check                      Test every package including uncommited changes and exit
+    -s, --scan [COMMIT]              Scan new commits (default) or specific commits
+Indexer options:
+    -a, --[no-]amend                 Reindex existing versions
     -i, --ignore PATH                Don't check or index any file starting with PATH
-    -n, --name NAME                  Set the name shown in ReaPack for this repository
     -U, --url-template TEMPLATE=auto Set the template for implicit download links
     -o, --output FILE=./index.xml    Set the output filename and path for the index
+Repository metadata:
+    -n, --name NAME                  Set the name shown in ReaPack for this repository
     -l, --link LINK                  Add or remove a website link
         --donation-link LINK         Add or remove a donation link
         --ls-links                   Display the link list then exit
     -A, --about=FILE                 Set the about content from a file
         --remove-about               Remove the about content from the index
         --dump-about                 Dump the raw about content in RTF and exit
+Misc options:
         --[no-]progress              Enable or disable progress information
     -V, --[no-]verbose               Activate diagnosis messages
     -C, --[no-]commit                Select whether to commit the modified index
@@ -50,6 +54,9 @@ Options:
     -h, --help                       Prints this help
 ```
 
+A getting started guide and packaging documentation are available in
+the [wiki](https://github.com/cfillion/reapack-index/wiki).
+
 ### Configuration
 
 Options can be specified from the command line or stored in configuration files.
@@ -60,112 +67,3 @@ The settings are applied in the following order:
 - ~/.reapack-index.conf (`~` = home directory)
 - ./.reapack-index.conf (`.` = repository root)
 - command line
-
-## Packaging Documentation
-
-This indexer uses metadata found at the start of the files to generate the
-database in ReaPack format.
-See also [MetaHeader](https://github.com/cfillion/metaheader)'s documentation.
-
-All tags are optional unless explicitely marked as required.
-
-### Package type by extension:
-
-- `.lua`, `.eel`, `.py`: ReaScripts
-- `.jsfx`: JS effects
-- `.ext`: Native extensions (all included files must be explicitely defined)
-
-### Package Tags
-
-These tags affects an entire package. Changes to any of those tags are
-applied immediately and may affect released versions.
-
-**@noindex**
-
-Disable indexing for this file. Set this on included files that
-should not be distributed alone.
-
-```
-@noindex
-
-NoIndex: true
-```
-
-**@version** [required]
-
-The current package version.
-Value must contain between one and four groups of digits.
-
-```
-@version 1.0
-@version 1.2pre3
-
-Version: 0.2015.12.25
-```
-
-### Version Tags
-
-These tags are specific to a single package version. You may still edit them
-after a release by running the indexer with the `--amend` option.
-
-**@author**
-
-```
-@author cfillion
-
-Author: Christian Fillion
-```
-
-**@changelog**
-
-```
-@changelog
-  Documented the metadata syntax
-  Added support for deleted scripts
-
-Changelog:
-  Added an alternate syntax for metadata tags
-```
-
-**@provides**
-
-Add additional files to the package. This is also used to add platform restrictions
-or set a custom download url (by default the download url is based on the "origin"
-git remote).  These files will be installed/updated together with the package.
-
-```
-@provides unicode.dat
-
-Provides:
-  Images/background.png
-  Images/fader_small.png
-  Images/fader_big.png
-
-@provides
-  [windows] reaper_extension.dll http://mysite.com/download/$version/$path
-```
-
-List of supported platform strings:
-- `windows`: All versions of Windows
-- `win32`: Windows 32-bit
-- `win64`: Windows 64-bit
-- `darwin`: All versions of OS X
-- `darwin32`: OS X 32-bit
-- `darwin64`: OS X 64-bit
-
-The following variables will be interpolated if found in the URL:
-- `$path`: The path of the file relative to the package
-- `$commit`: The hash of the commit being indexed or "master" if unavailable
-- `$version`: The version of the package being indexed
-
-Platform restriction and custom url can be set for the package itself,
-either by using its file name or a dot:
-
-```
--- this is a lua script named `hello_osx.lua`
--- @provides
---   [darwin] hello_osx.lua
-
--- @provides
---   [darwin] .
-```

data/Rakefile

@@ -3,6 +3,6 @@ require 'rake/testtask'
 
 task :default => [:test]
 Rake::TestTask.new do |t|
-  t.test_files = FileList['test/test_*.rb']
+  t.test_files = FileList['test/**/test_*.rb']
   t.warning = false
 end

data/lib/reapack/index.rb

@@ -14,58 +14,47 @@ require 'rugged'
 require 'shellwords'
 require 'time'
 
+require 'reapack/index/cdetector'
 require 'reapack/index/cli'
 require 'reapack/index/cli/options'
+require 'reapack/index/git'
 require 'reapack/index/metadata'
 require 'reapack/index/named_node'
 require 'reapack/index/package'
 require 'reapack/index/parsers'
+require 'reapack/index/provides'
+require 'reapack/index/source'
 require 'reapack/index/version'
 
 class ReaPack::Index
   Error = Class.new RuntimeError
 
-  FILE_TYPES = {
-    'lua'  => :script,
-    'eel'  => :script,
-    'py'   => :script,
-    'ext'  => :extension,
-    'jsfx' => :effect,
+  PKG_TYPES = {
+    script: %w{lua eel py},
+    extension: %w{ext},
+    effect: %w{jsfx},
+    data: %w{data},
   }.freeze
 
   WITH_MAIN = [:script, :effect].freeze
 
-  PROVIDES_REGEX = /
-    \A
-    ( \[ \s* (?<platform> .+? ) \s* \] )?
-    \s*
-    (?<file> .+?)
-    ( \s+ (?<url> (?:file|https?):\/\/.+ ) )?
-    \z
-  /x.freeze
-
   PROVIDES_VALIDATOR = proc {|value|
     begin
-      files = value.lines.map {|l|
-        m = l.chomp.match PROVIDES_REGEX
-        Source.validate_platform m[:platform]
-        [m[:platform], m[:file]]
-      }
-      dup = files.detect {|f| files.count(f) > 1 }
-      "duplicate file (%s)" % dup[1] if dup
+      Provides.parse_each(value).to_a and nil
     rescue Error => e
       e.message
     end
-  }.freeze
+  }
 
   HEADER_RULES = {
     # package-wide tags
-    :version => /\A(?:[^\d]*\d{1,4}[^\d]*){1,4}\z/,
+    :version => [
+      MetaHeader::REQUIRED, MetaHeader::VALUE, MetaHeader::SINGLELINE, /\d/],
 
     # version-specific tags
-    :author => [MetaHeader::OPTIONAL, /\A[^\n]+\z/],
-    :changelog => [MetaHeader::OPTIONAL, /.+/],
-    :provides => [MetaHeader::OPTIONAL, PROVIDES_VALIDATOR]
+    :author => [MetaHeader::VALUE, MetaHeader::SINGLELINE],
+    :changelog => [MetaHeader::VALUE],
+    :provides => [MetaHeader::VALUE, PROVIDES_VALIDATOR]
   }.freeze
 
   FS_ROOT = File.expand_path('/').freeze
@@ -73,28 +62,51 @@ class ReaPack::Index
   attr_reader :path, :url_template
   attr_accessor :amend, :files, :time
 
-  def self.type_of(path)
-    ext = File.extname(path)[1..-1]
-    FILE_TYPES[ext]
-  end
+  class << self
+    def is_type?(input)
+      PKG_TYPES.has_key? input&.to_sym
+    end
 
-  def self.validate_file(path)
-    mh = MetaHeader.from_file path
-    return if mh[:noindex]
+    def type_of(path)
+      # don't treat files in the root directory as packages
+      # because they don't have a category
+      return if File.dirname(path) == '.'
 
-    mh.validate HEADER_RULES
+      ext = File.extname(path)[1..-1]
+      PKG_TYPES.find {|_, v| v.include? ext }&.first if ext
+    end
+
+    alias :is_package? :type_of
+
+    def resolve_type(input)
+      PKG_TYPES
+        .find {|name, exts| input.to_sym == name || exts.include?(input.to_s) }
+        &.first
+    end
   end
 
   def initialize(path)
     @amend = false
     @changes = {}
+    @changed_nodes = []
     @files = []
     @path = path
 
+    @cdetector = ConflictDetector.new
+
     if File.exist? path
-      # noblanks: don't preserve the original white spaces
-      # so we always output a neat document
-      @doc = File.open(path) {|file| Nokogiri::XML file, &:noblanks }
+      begin
+        # noblanks: don't preserve the original white spaces
+        # so we always output a neat document
+        @doc = File.open(path) {|file| Nokogiri::XML file, &:noblanks }
+      rescue Nokogiri::XML::SyntaxError
+      end
+
+      unless @doc&.root&.name == 'index'
+        raise Error, "'#{path}' is not a ReaPack index file"
+      end
+
+      @cdetector.load_xml @doc.root
     else
       @dirty = true
       @is_new = true
@@ -113,9 +125,12 @@ class ReaPack::Index
     type = self.class.type_of path
     return unless type
 
-    mh = MetaHeader.new contents
+    # variables to restore if an error occur
+    backups = Hash[[:@doc, :@cdetector].map {|var|
+      [var, instance_variable_get(var).clone]
+    }]
 
-    backup = @doc.root.dup
+    mh = MetaHeader.new contents
 
     if mh[:noindex]
       remove path
@@ -123,13 +138,12 @@ class ReaPack::Index
     end
 
     if errors = mh.validate(HEADER_RULES)
-      prefix = errors.size == 1 ? "\x20" : "\n\x20\x20"
-      raise Error, "invalid metadata:%s" %
-        [prefix + errors.join(prefix)]
+      raise Error, errors.join("\n")
     end
 
-    cat, pkg = find path
-    pkg.type = type.to_s
+    cat, pkg = package_for path
+
+    cselector = @cdetector[pkg.type = type, path]
 
     pkg.version mh[:version] do |ver|
       next unless ver.is_new? || @amend
@@ -138,48 +152,59 @@ class ReaPack::Index
       @currentVersion = ver.name
 
       ver.author = mh[:author]
-      ver.time = @time if @time
+      ver.time = @time if @time && ver.is_new?
       ver.changelog = mh[:changelog]
 
       ver.replace_sources do
-        sources = parse_provides mh[:provides], path
+        cselector.clear
+        sources = parse_provides mh[:provides], pkg
 
         if WITH_MAIN.include?(type) && sources.none? {|src| src.file.nil? }
           # add the package itself as a source
-          sources.unshift Source.new nil, nil, make_url(path)
+          src = Source.new make_url(path)
+          sources.unshift src
+
+          cselector.push src.platform, path
         end
 
         sources.each {|src| ver.add_source src }
       end
     end
 
+    if cons = cselector.resolve
+      raise Error, cons.first
+    end
+
     log_change 'new category', 'new categories' if cat.is_new?
 
-    if pkg.is_new?
-      log_change 'new package'
-    elsif pkg.modified?
-      log_change 'modified package'
+    if pkg.modified? && !@changed_nodes.include?(pkg.node)
+      log_change "#{pkg.is_new? ? 'new' : 'modified'} package"
+      @changed_nodes << pkg.node
     end
 
     pkg.versions.each {|ver|
-      if ver.is_new?
-        log_change 'new version'
-      elsif ver.modified?
-        log_change 'modified version'
+      if ver.modified? && !@changed_nodes.include?(ver.node)
+        log_change "#{ver.is_new? ? 'new' : 'modified'} version"
+        @changed_nodes << ver.node
       end
     }
+
+    bump_commit
   rescue Error
-    @doc.root = backup
+    backups.each {|var, value| instance_variable_set var, value }
     raise
   end
   
   def remove(path)
-    cat, pkg = find path, false
+    cat, pkg = package_for path, false
     return unless pkg
 
+    @cdetector[pkg.type, path].clear
+
     pkg.remove
     cat.remove if cat.empty?
 
+    bump_commit
     log_change 'removed package'
   end
 
@@ -221,7 +246,7 @@ class ReaPack::Index
     uri.normalize!
 
     unless (uri.request_uri || uri.path).include? '$path'
-      raise Error, "$path placeholder is missing: #{tpl}"
+      raise Error, "missing $path placeholder in '#{tpl}'"
     end
 
     unless %w{http https file}.include? uri.scheme
@@ -230,7 +255,7 @@ class ReaPack::Index
 
     @url_template = uri.to_s.freeze
   rescue Addressable::URI::InvalidURIError
-    raise Error, "invalid URL or scheme: #{tpl}"
+    raise Error, "invalid template '#{tpl}'"
   end
 
   def version
@@ -243,7 +268,7 @@ class ReaPack::Index
 
   def name=(newName)
     if !/\A[^~#%&*{}\\:<>?\/+|"]+\Z/.match(newName) || /\A\.+\Z/.match(newName)
-      raise Error, "Invalid name: '#{newName}'"
+      raise Error, "invalid name '#{newName}'"
     end
 
     oldName = name
@@ -252,19 +277,12 @@ class ReaPack::Index
   end
 
   def commit
-    @doc.root[:commit]
+    @commit ||= @doc.root[:commit]
   end
 
-  def commit=(sha1)
-    if sha1.nil?
-      @doc.root.remove_attribute 'commit'
-    else
-      @doc.root['commit'] = sha1
-    end
-  end
+  attr_writer :commit
 
   def write(path)
-    @doc.root.element_children.each {|n| sort n if n.name == 'category' }
     sort @doc.root
 
     FileUtils.mkdir_p File.dirname(path)
@@ -295,6 +313,29 @@ class ReaPack::Index
     list.join ', '
   end
 
+  def make_url(path, template = nil)
+    unless template
+      unless @url_template
+        raise Error, 'unable to generate download links: empty url template'
+      end
+
+      unless @files.include? path
+        raise Error, "file not found '#{path}'"
+      end
+    end
+
+    (template || @url_template)
+      .sub('$path', path)
+      .sub('$commit', commit || 'master')
+      .sub('$version', @currentVersion.to_s)
+  end
+
+  def self.expand(filepath, basedir)
+    expanded = File.expand_path filepath, FS_ROOT + basedir
+    expanded[0...FS_ROOT.size] = ''
+    expanded
+  end
+
 private
   def log_change(desc, plural = nil)
     @dirty = true
@@ -303,62 +344,71 @@ private
     @changes[desc][0] += 1
   end
 
-  def dirname(path)
-    name = File.dirname path
-    name == '.' ? nil : name
-  end
-
-  def find(path, create = true)
-    cat_name = dirname(path) || 'Other'
+  def package_for(path, create = true)
+    cat_name = File.dirname path
     pkg_name = File.basename path
 
-    cat = Category.get cat_name, @doc.root, create
-    pkg = Package.get pkg_name, cat && cat.node, create
+    cat = Category.fetch cat_name, @doc.root, create
+    pkg = Package.fetch pkg_name, cat&.node, create
 
     [cat, pkg]
   end
 
-  def make_url(path, template = nil)
-    if template.nil?
-      raise Error, 'unable to generate a download link' \
-        ' – the url template is unset' unless @url_template
-
-      raise Error, "#{path}: No such file or directory" unless @files.include? path
-    end
+  def parse_provides(provides, pkg)
+    pathdir = Pathname.new pkg.category
 
-    (template || @url_template)
-      .sub('$path', path)
-      .sub('$commit', commit || 'master')
-      .sub('$version', @currentVersion || '0.0')
-  end
+    Provides.parse_each(provides).map {|line|
+      line.file_pattern = pkg.name if line.file_pattern == '.'
 
-  def parse_provides(provides, path)
-    basename = File.basename path
-    basedir = dirname path
+      expanded = self.class.expand line.file_pattern, pkg.category
+      cselector = @cdetector[line.type || pkg.type, pkg.path]
 
-    provides.to_s.lines.map {|line|
-      line.chomp!
+      if expanded == pkg.path
+        # always resolve path even when an url template is set
+        files = [expanded]
+      elsif line.url_template.nil?
+        files = @files.select {|f| File.fnmatch expanded, f, File::FNM_PATHNAME }
+        raise Error, "file not found '#{line.file_pattern}'" if files.empty?
+      else
+        # use the relative path for external urls
+        files = [line.file_pattern]
+      end
 
-      m = line.match PROVIDES_REGEX
+      files.map {|file|
+        src = Source.new make_url(file, line.url_template)
+        src.platform = line.platform
+        src.type = line.type
 
-      platform, file, url_tpl = m[:platform], m[:file], m[:url]
-      file = nil if file == basename || file == '.'
+        cselector.push src.platform, line.url_template ? expanded : file
 
-      if file.nil?
-        url = make_url path, url_tpl
-      elsif url_tpl.nil?
-        normalized = File.expand_path file, FS_ROOT + basedir.to_s
-        url = make_url normalized[FS_ROOT.size..-1]
-      else
-        url = make_url file, url_tpl
-      end
+        if file != pkg.path
+          if line.url_template
+            src.file = file
+          else
+            src.file = Pathname.new(file).relative_path_from(pathdir).to_s
+          end
+        end
 
-      Source.new platform, file, url
-    }
+        src
+      }
+    }.flatten
   end
 
   def sort(node)
+    node.children.each {|n| sort n }
+
+    return if node.name == Package.tag
     sorted = node.children.sort_by{|n| n[:name].to_s }.sort_by {|n| n.name }
     sorted.each {|n| node << n }
   end
+
+  def bump_commit
+    sha1 = commit()
+
+    if sha1.nil?
+      @doc.root.remove_attribute 'commit'
+    else
+      @doc.root['commit'] = sha1
+    end
+  end
 end