apt-dists-merge 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 661b0b8dd9099c4af744f1d9026fa0f2474af1f81be6247a5463f4d862b003fb
+  data.tar.gz: 0f2d7ac3478faf2f48d9f5c6fc3832062a9a440705b275fb6729a541405b687e
+SHA512:
+  metadata.gz: 49d26cfb67d54dc6fee5ad44a0b6939c4672f04f0bef30cd7d042b50c002afdf10a64e3f46147f95046de48d3facddd8f89a9a46714703801528f087c4c3e06d
+  data.tar.gz: f95558f6ddc923b0a39fc42616eaada2dd937c40319c4ce0be1d4eab05656f1b3437739ed1739d03c52c7f449e1ed86de471ebb121b69162524133831886acc7

data/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright 2021 Sutou Kouhei <kou@clear-code.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/NEWS.md

@@ -0,0 +1,5 @@
+# NEWS
+
+## 1.0.0 - 2021-09-26
+
+Initial release.

data/README.md

@@ -0,0 +1,42 @@
+# APT dists merge
+
+## Description
+
+APT dists merge provides a tool and a library to merge `dists/` contents for APT repository.
+
+Generally, you should use more general APT repository management tool such as [aptly](https://www.aptly.info/) and [reprepro](https://salsa.debian.org/brlink/reprepro).
+
+APT dists merge is useful when you want to manage `pool/` by yourself and add `.deb`s incrementally without keeping all `.deb`s for the target APT repository on local.
+
+Use cases:
+
+  * [Apache Arrow](https://github.com/apache/arrow/blob/master/dev/release/binary-task.rb)
+  * [Groonga](https://github.com/groonga/packages.groonga.org)
+  * [Red Data Tools](https://github.com/red-data-tools/packages.red-data-tools.org)
+
+## Install
+
+```bash
+gem install apt-dists-merge
+```
+
+## Usage
+
+Tool:
+
+```bash
+apt-dists-merge --base base/dists/ --new new/dists/ --output merged/dists/
+```
+
+Library:
+
+```ruby
+require "apt-dists-merge"
+
+merger = APTDistsMerge::Merger.new("base/dists/", "new/dists/")
+merger.merge("merged/dists/")
+```
+
+## License
+
+The MIT license. See `LICENSE.txt` for details.

data/bin/apt-dists-merge

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/apt-dists-merge"
+
+command_line = APTDistsMerge::CommandLine.new
+exit(command_line.run(ARGV))

data/lib/apt-dists-merge/command-line.rb

@@ -0,0 +1,63 @@
+require "digest"
+require "optparse"
+require "zlib"
+
+module APTDistsMerge
+  class CommandLine
+    def initialize(output=nil)
+      @base_dir = nil
+      @incomping_dir = nil
+      @output_dir = nil
+      @output = output || "-"
+    end
+
+    def run(args)
+      catch do |tag|
+        open_output do |output|
+          parse_args(args, output, tag)
+          process
+        end
+      end
+    end
+
+    private
+    def open_output(&block)
+      case @output
+      when "-"
+        yield($stdout)
+      when String
+        File.open(@output, "w", &block)
+      else
+        yield(@output)
+      end
+    end
+
+    def parse_args(args, output, tag)
+      parser = OptionParser.new
+      parser.banner += " BASE_DIR INCOMING_DIR OUTPUT_DIR"
+      parser.on("--version",
+                "Show version and exit") do
+        output.puts(VERSION)
+        throw(tag, true)
+      end
+      parser.on("--help",
+                "Show this message and exit") do
+        output.puts(parser.help)
+        throw(tag, true)
+      end
+      args = parser.parse!(args.dup)
+      if args.size != 3
+        $stderr.puts(parser.help)
+        throw(tag, false)
+      end
+      @base_dir = args[0]
+      @incoming_dir = args[1]
+      @output_dir = args[2]
+    end
+
+    def process
+      merger = Merger.new(@base_dir, @incoming_dir, @output_dir)
+      merger.merge
+    end
+  end
+end

data/lib/apt-dists-merge/merger.rb

@@ -0,0 +1,321 @@
+require "fileutils"
+
+module APTDistsMerge
+  class Merger
+    def initialize(base_dir, incoming_dir, merged_dir)
+      @base_dir = base_dir
+      @incoming_dir = incoming_dir
+      @merged_dir = merged_dir
+    end
+
+    def merge
+      components = (detect_components(@base_dir) |
+                    detect_components(@incoming_dir))
+      components.each do |component|
+        return false unless merge_component(component)
+      end
+      return false unless merge_release
+      true
+    end
+
+    private
+    def detect_extensions(base_path)
+      Dir.glob("#{base_path}*").collect do |path|
+        path.gsub(/\A#{Regexp.escape(base_path)}/, "")
+      end
+    end
+
+    def read_data(path)
+      if File.exist?(path)
+        File.read(path)
+      elsif File.exist?("#{path}.gz")
+        real_path = "#{path}.gz"
+        Zlib::GzipReader.open(real_path) do |input|
+          input.read
+        end
+      elsif File.exist?("#{path}.xz")
+        real_path = "#{path}.xz"
+        IO.popen(["xz", "--decompress", "--stdout", real_path]) do |input|
+          input.read
+        end
+      else
+        nil
+      end
+    end
+
+    def write_data(data, path, extensions=nil)
+      FileUtils.mkdir_p(File.dirname(path))
+      extensions ||= [""]
+      extensions.each do |extension|
+        case extension
+        when ".gz"
+          Zlib::GzipWriter.open("#{path}#{extension}") do |output|
+            output.write(data)
+          end
+        when ".xz"
+          IO.popen(["xz", "--stdout"], "r+") do |xz|
+            writer = Thread.new do
+              xz.write(data)
+              xz.close_write
+            end
+            File.open("#{path}#{extension}", "wb") do |output|
+              reader = Thread.new do
+                IO.copy_stream(xz, output)
+              end
+              writer.join
+              reader.join
+            end
+          end
+        else
+          File.open(path, "w") do |output|
+            output.write(data)
+          end
+        end
+      end
+    end
+
+    def release_path(dir)
+      "#{dir}/Release"
+    end
+
+    def source_release_path(dir, component)
+      "#{dir}/#{component}/source/Release"
+    end
+
+    def sources_path(dir, component)
+      "#{dir}/#{component}/source/Sources"
+    end
+
+    def contents_path(dir, component, arch)
+      "#{dir}/#{component}/Contents-#{arch}"
+    end
+
+    def packages_path(dir, component, arch)
+      "#{dir}/#{component}/binary-#{arch}/Packages"
+    end
+
+    def binary_release_path(dir, component, arch)
+      "#{dir}/#{component}/binary-#{arch}/Release"
+    end
+
+    def detect_architectures(dir)
+      data = read_data(release_path(dir))
+      return [] if data.nil?
+      data.each_line do |line|
+        case line
+        when /\AArchitectures:\s*/
+          return Regexp.last_match.post_match.split(/\s+/)
+        end
+      end
+      []
+    end
+
+    def detect_components(dir)
+      components = []
+      Dir.open(dir) do |d|
+        d.each do |path|
+          full_path = "#{dir}/#{path}"
+          next if path.start_with?(".")
+          next unless File.directory?(full_path)
+          components << path
+        end
+      end
+      components
+    end
+
+    def merge_component(component)
+      architectures = (detect_architectures(@base_dir) |
+                       detect_architectures(@incoming_dir))
+      architectures.each do |arch|
+        return false unless merge_architecture(component, arch)
+      end
+      if File.exist?(source_release_path(@base_dir, component)) or
+        File.exist?(source_release_path(@incoming_dir, component))
+        return false unless merge_source(component)
+      end
+      true
+    end
+
+    def merge_architecture(component, arch)
+      return false unless merge_contents(component, arch)
+      return false unless merge_packages(component, arch)
+      return false unless merge_binary_release(component, arch)
+      true
+    end
+
+    def merge_contents(component, arch)
+      base_path = contents_path(@base_dir, component, arch)
+      incoming_path = contents_path(@incoming_dir, component, arch)
+      base = read_data(base_path)
+      incoming = read_data(incoming_path)
+      merged = (base.lines | incoming.lines).sort
+      merged_path = contents_path(@merged_dir, component, arch)
+      write_data(merged.join,
+                 merged_path,
+                 detect_extensions(base_path) |
+                 detect_extensions(incoming_path))
+      true
+    end
+
+    def parse_deb822(content)
+      data = {}
+      key = nil
+      content.each_line do |line|
+        case line
+        when /\A /
+          data[key] << "\n" << line
+        else
+          key, value = line.split(/:\s*/, 2)
+          data[key] = value.chomp
+        end
+      end
+      data
+    end
+
+    def read_packages(path)
+      read_data(path).split("\n\n").collect do |content|
+        [parse_deb822(content), content]
+      end
+    end
+
+    def debian_version_compare(a, b)
+      if system("dpkg", "--compare-versions", a, "eq", b)
+        0
+      elsif system("dpkg", "--compare-versions", a, "lt", b)
+        -1
+      else
+        1
+      end
+    end
+
+    def merge_packages_data(base, incoming)
+      (base + incoming).sort do |a, b|
+        a_metadata = a[0]
+        b_metadata = b[0]
+        a_package = a_metadata["Package"]
+        b_package = b_metadata["Package"]
+        package_compare = (a_package <=> b_package)
+        if package_compare == 0
+          debian_version_compare(a_metadata["Version"], b_metadata["Version"])
+        else
+          if a_package.start_with?(b_package)
+            -1
+          else
+            package_compare
+          end
+        end
+      end
+    end
+
+    def merge_packages(component, arch)
+      base_path = packages_path(@base_dir, component, arch)
+      incoming_path = packages_path(@incoming_dir, component, arch)
+      base = read_packages(base_path)
+      incoming = read_packages(incoming_path)
+      merged = merge_packages_data(base, incoming)
+      merged_path = packages_path(@merged_dir, component, arch)
+      write_data(merged.collect {|_, content| "#{content}\n\n"}.join,
+                 merged_path,
+                 detect_extensions(base_path) |
+                 detect_extensions(incoming_path))
+      true
+    end
+
+    def merge_binary_release(component, arch)
+      base_path = binary_release_path(@base_dir, component, arch)
+      incoming_path = binary_release_path(@incoming_dir, component, arch)
+      base = read_data(base_path)
+      incoming = read_data(incoming_path)
+      merged = incoming || base
+      merged_path = binary_release_path(@merged_dir, component, arch)
+      write_data(merged,
+                 merged_path,
+                 detect_extensions(base_path) |
+                 detect_extensions(incoming_path))
+      true
+    end
+
+    def merge_source(component)
+      return false unless merge_sources(component)
+      return false unless merge_source_release(component)
+      true
+    end
+
+    def merge_sources(component)
+      base_path = sources_path(@base_dir, component)
+      incoming_path = sources_path(@incoming_dir, component)
+      base = read_packages(base_path)
+      incoming = read_packages(incoming_path)
+      merged = merge_packages_data(base, incoming)
+      merged_path = sources_path(@merged_dir, component)
+      write_data(merged.collect {|_, content| "#{content}\n\n"}.join,
+                 merged_path,
+                 detect_extensions(base_path) |
+                 detect_extensions(incoming_path))
+      true
+    end
+
+    def merge_source_release(component)
+      base_path = source_release_path(@base_dir, component)
+      incoming_path = source_release_path(@incoming_dir, component)
+      base = read_data(base_path)
+      incoming = read_data(incoming_path)
+      merged = incoming || base
+      merged_path = source_release_path(@merged_dir, component)
+      write_data(merged,
+                 merged_path,
+                 detect_extensions(base_path) |
+                 detect_extensions(incoming_path))
+      true
+    end
+
+    def read_release(path)
+      parse_deb822(read_data(path))
+    end
+
+    def generate_checksums(dir, files, digest_class)
+      checksums = files.collect do |file|
+        checksum = digest_class.file(file).to_s
+        size = File.size(file)
+        relative_path = file.gsub(/\A#{Regexp.escape(dir)}\//, "")
+        " %s %16d %s" % [checksum, size, relative_path]
+      end
+      checksums.join("\n")
+    end
+
+    def merge_release
+      base = read_release(release_path(@base_dir))
+      incoming = read_release(release_path(@incoming_dir))
+      architectures = base["Architectures"].split |
+                      incoming["Architectures"].split
+      components = base["Components"].split |
+                   incoming["Components"].split
+      files = Dir.glob("#{@merged_dir}/**/*")
+      files = files.reject do |file|
+        File.directory?(file)
+      end
+      files = files.sort
+      merged = <<-RELEASE
+Architectures: #{architectures.sort.join(" ")}
+Codename: #{incoming["Codename"]}
+Components: #{components.sort.join(" ")}
+Date: #{incoming["Date"]}
+Description: #{incoming["Description"]}
+Label: #{incoming["Label"]}
+Origin: #{incoming["Origin"]}
+Suite: #{incoming["Suite"]}
+MD5Sum:
+#{generate_checksums(@merged_dir, files, Digest::MD5)}
+SHA1:
+#{generate_checksums(@merged_dir, files, Digest::SHA1)}
+SHA256:
+#{generate_checksums(@merged_dir, files, Digest::SHA256)}
+SHA512:
+#{generate_checksums(@merged_dir, files, Digest::SHA512)}
+      RELEASE
+      merged_path = release_path(@merged_dir)
+      write_data(merged, merged_path)
+      true
+    end
+  end
+end

data/lib/apt-dists-merge/version.rb

@@ -0,0 +1,3 @@
+module APTDistsMerge
+  VERSION = "1.0.0"
+end

data/lib/apt-dists-merge.rb

@@ -0,0 +1,3 @@
+require_relative "apt-dists-merge/command-line"
+require_relative "apt-dists-merge/merger"
+require_relative "apt-dists-merge/version"

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: apt-dists-merge
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Sutou Kouhei
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-09-26 00:00:00.000000000 Z
+dependencies: []
+description: Generally, you should use more general APT repository management tool
+  such as [aptly](https://www.aptly.info/) and [reprepro](https://salsa.debian.org/brlink/reprepro).
+email:
+- kou@clear-code.com
+executables:
+- apt-dists-merge
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- NEWS.md
+- README.md
+- bin/apt-dists-merge
+- lib/apt-dists-merge.rb
+- lib/apt-dists-merge/command-line.rb
+- lib/apt-dists-merge/merger.rb
+- lib/apt-dists-merge/version.rb
+homepage: https://github.com/red-data-tools/apt-dists-merge
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.0.dev
+signing_key: 
+specification_version: 4
+summary: APT dists merge provides a tool and a library to merge `dists/` contents
+  for APT repository.
+test_files: []