zip-merge 7.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a859c92350627feaf0b728b47e735675f19c339d50c6bf709a5c516e6fd252ec
+  data.tar.gz: 54d03cd74d3ec83b9050bdcad173320f5a9c00dd49a657a04f736f92c3ce53bc
+SHA512:
+  metadata.gz: 96e218946eb50b84c11e3d1fff6354626fd44f50d1672e32275aa3eb18e960821566845a0a4fd6adcd1a68a7c2be9d2de6255c2933da91b266105309e3369b20
+  data.tar.gz: 882dadf7d82282213faf7d43691eee6f4382a14eb261c5b4aeb7659db609c9532b1c86a688a5bdd2f0c034a1c1eaf37c8db98f31536f726c65f490488763e4d5

data/lib/zip/merge/version.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Zip
+  module Merge
+    module Version
+      VERSION = "7.0.0"
+    end
+
+    VERSION = Version::VERSION
+  end
+end

data/lib/zip/merge.rb

@@ -0,0 +1,311 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "zlib"
+require "tree_haver"
+require_relative "merge/version"
+
+module Zip
+  module Merge
+    LOCAL = 0x04034b50
+    CENTRAL = 0x02014b50
+    EOCD = 0x06054b50
+    DOS_EPOCH = [0, 0, 0x21, 0].pack("C*")
+
+    RenderError = Class.new(StandardError) do
+      attr_reader :diagnostic
+
+      def initialize(diagnostic)
+        @diagnostic = diagnostic
+        super(diagnostic.message)
+      end
+    end
+
+    module_function
+
+    def parse_zip_inventory(source)
+      bytes = source.b
+      central = scan_central_directory(bytes)
+      locals = scan_local_headers(bytes, central[:records])
+      entries = central[:records].map do |name, record|
+        local = locals.fetch(name)
+        TreeHaver::ZipArchiveEntry.new(
+          path: name,
+          normalized_path: normalize_zip_path(name),
+          directory: name.end_with?("/"),
+          compression: compression_name(record[:method]),
+          compressed_size: record[:compressed_size],
+          uncompressed_size: record[:uncompressed_size],
+          crc32: "%08x" % record[:crc32],
+          local_header_range: TreeHaver::ByteRange.new(start_byte: record[:local_offset], end_byte: local[:data_start]),
+          data_range: TreeHaver::ByteRange.new(start_byte: local[:data_start], end_byte: local[:data_start] + record[:compressed_size]),
+          central_directory_range: record[:range]
+        )
+      end.sort_by { |entry| entry.local_header_range.start_byte }
+
+      TreeHaver::ZipFamilyReport.new(
+        archive: TreeHaver::ZipArchiveInfo.new(format: "zip", schema: "zip.ksy", entry_count: entries.length, central_directory_range: central[:range]),
+        entries: entries,
+        member_decisions: [],
+        unsafe_entries: unsafe_entries(entries, central[:records]),
+        merge_report: empty_report
+      )
+    end
+
+    def plan_zip_merge(ancestor, current, incoming)
+      report = TreeHaver::ZipFamilyReport.new(
+        archive: incoming.archive,
+        entries: incoming.entries,
+        member_decisions: [],
+        unsafe_entries: incoming.unsafe_entries || [],
+        merge_report: empty_report
+      )
+      ancestor_entries = entries_by_path(ancestor.entries)
+      current_entries = entries_by_path(current.entries)
+      incoming_entries = entries_by_path(incoming.entries)
+      unsafe_by_path = report.unsafe_entries.to_h { |entry| [entry.normalized_path, entry] }
+
+      (ancestor_entries.keys | current_entries.keys | incoming_entries.keys).sort.each do |path|
+        ancestor_entry = ancestor_entries[path]
+        current_entry = current_entries[path]
+        incoming_entry = incoming_entries[path]
+        if unsafe_by_path[path]
+          unsafe = unsafe_by_path[path]
+          report.member_decisions << TreeHaver::ZipMemberDecision.new(normalized_path: path, operation: "reject", disposition: "unsafe", reason: unsafe.reason)
+          report.merge_report.diagnostics << diagnostic(unsafe.category, schema_path(path), unsafe.reason)
+        elsif current_entry.nil? && incoming_entry
+          decision(report, path, "add", "requires_renderer", "member exists only in incoming archive")
+        elsif current_entry && incoming_entry.nil?
+          decision(report, path, "delete", "requires_renderer", "member was removed from incoming archive")
+        elsif ancestor_entry && same_entry?(current_entry, ancestor_entry) && same_entry?(incoming_entry, ancestor_entry)
+          report.member_decisions << TreeHaver::ZipMemberDecision.new(normalized_path: path, operation: "preserve", disposition: "safe", reason: "member is unchanged from ancestor")
+          report.merge_report.preserved_ranges.concat([current_entry.local_header_range, current_entry.data_range])
+        elsif (family = nested_family(path))
+          report.member_decisions << TreeHaver::ZipMemberDecision.new(normalized_path: path, operation: "delegate", disposition: "requires_renderer", nested_family: family, reason: "structured member can be merged by a nested family before ZIP rendering")
+          report.merge_report.nested_dispatches << TreeHaver::BinaryNestedDispatch.new(schema_path: "#{schema_path(path)}/data", family: family, status: "planned")
+          report.merge_report.rewritten_nodes << schema_path(path)
+          report.merge_report.checksum_updates << "#{schema_path(path)}/crc32"
+        else
+          decision(report, path, "rewrite", "requires_renderer", "member bytes or metadata changed")
+          report.merge_report.checksum_updates << "#{schema_path(path)}/crc32"
+        end
+        report.merge_report.matched_schema_paths << schema_path(path)
+      end
+      unless report.merge_report.rewritten_nodes.empty? && report.merge_report.checksum_updates.empty?
+        report.merge_report.rewritten_nodes << "/central_directory"
+        report.merge_report.checksum_updates.concat(["/central_directory/size", "/central_directory/offset"])
+      end
+      report
+    end
+
+    def render_with_raw_preservation(source:, plan:, member_bytes: {}, compression: 0)
+      raise render_error("unsupported_compression", "/render/options/compression", "unsupported raw-preserving compression method") unless [0, 8].include?(compression)
+
+      source = source.b
+      source_inventory = parse_zip_inventory(source)
+      central = scan_central_directory(source)
+      source_entries = entries_by_path(source_inventory.entries)
+      raw_ranges = raw_local_record_ranges(source, source_entries)
+      output = +"".b
+      central_records = []
+      entries = entries_by_path(plan.entries)
+      plan.member_decisions.each do |member|
+        entry = entries[member.normalized_path]
+        case member.operation
+        when "reject"
+          raise render_error("rejected_member", schema_path(member.normalized_path), member.reason)
+        when "delete"
+          next
+        when "preserve"
+          source_entry = source_entries.fetch(member.normalized_path)
+          validate_raw_preserve_entry!(source, central, source_entry)
+          range = raw_ranges.fetch(member.normalized_path)
+          offset = output.bytesize
+          output << source.byteslice(range.start_byte...range.end_byte)
+          central_records << central_record_from_entry(source_entry, offset)
+        when "add", "rewrite", "delegate"
+          content = member_bytes.fetch(member.normalized_path)
+          rendered, record = rendered_local_record(entry, content.b, compression, output.bytesize)
+          output << rendered
+          central_records << record
+        else
+          raise "unsupported ZIP render operation #{member.operation.inspect}"
+        end
+      end
+      central_start = output.bytesize
+      central_records.each { |record| output << central_directory_record(record) }
+      central_size = output.bytesize - central_start
+      output << eocd_record(central_records.length, central_size, central_start)
+      report = parse_zip_inventory(output)
+      merge_report = plan.merge_report
+      merge_report.preserved_ranges = plan.member_decisions.filter_map { |member| raw_ranges[member.normalized_path] if member.operation == "preserve" }
+      [output, report, merge_report]
+    end
+
+    def new_stored_zip(entries)
+      output = +"".b
+      central = []
+      entries.keys.sort.each do |name|
+        rendered, record = rendered_local_record(path_entry(name, entries[name]), entries[name].b, 0, output.bytesize)
+        output << rendered
+        central << record
+      end
+      start = output.bytesize
+      central.each { |record| output << central_directory_record(record) }
+      output << eocd_record(central.length, output.bytesize - start, start)
+      output
+    end
+
+    def empty_report
+      TreeHaver::BinaryMergeReport.new(format: "zip", schema: "zip.ksy", matched_schema_paths: [], preserved_ranges: [], rewritten_nodes: [], checksum_updates: [], nested_dispatches: [], diagnostics: [])
+    end
+
+    def scan_central_directory(source)
+      eocd = source.bytesize - 22
+      eocd -= 1 while eocd >= 0 && source.byteslice(eocd, 4).unpack1("V") != EOCD
+      raise "missing ZIP end of central directory" if eocd.negative?
+      size = source.byteslice(eocd + 12, 4).unpack1("V")
+      offset = source.byteslice(eocd + 16, 4).unpack1("V")
+      comment_length = source.byteslice(eocd + 20, 2).unpack1("v")
+      records = {}
+      cursor = offset
+      while cursor < offset + size
+        raise "unexpected central directory record" unless source.byteslice(cursor, 4).unpack1("V") == CENTRAL
+        name_len = source.byteslice(cursor + 28, 2).unpack1("v")
+        extra_len = source.byteslice(cursor + 30, 2).unpack1("v")
+        comment_len = source.byteslice(cursor + 32, 2).unpack1("v")
+        name = source.byteslice(cursor + 46, name_len)
+        records[name] = {
+          range: TreeHaver::ByteRange.new(start_byte: cursor, end_byte: cursor + 46 + name_len + extra_len + comment_len),
+          flags: source.byteslice(cursor + 8, 2).unpack1("v"),
+          method: source.byteslice(cursor + 10, 2).unpack1("v"),
+          crc32: source.byteslice(cursor + 16, 4).unpack1("V"),
+          compressed_size: source.byteslice(cursor + 20, 4).unpack1("V"),
+          uncompressed_size: source.byteslice(cursor + 24, 4).unpack1("V"),
+          extra_length: extra_len,
+          comment_length: comment_len,
+          local_offset: source.byteslice(cursor + 42, 4).unpack1("V")
+        }
+        cursor = records[name][:range].end_byte
+      end
+      { range: TreeHaver::ByteRange.new(start_byte: offset, end_byte: offset + size), records: records, archive_comment: comment_length.positive? }
+    end
+
+    def scan_local_headers(source, records)
+      records.transform_values do |record|
+        cursor = record[:local_offset]
+        raise "unexpected ZIP local header" unless source.byteslice(cursor, 4).unpack1("V") == LOCAL
+        name_len = source.byteslice(cursor + 26, 2).unpack1("v")
+        extra_len = source.byteslice(cursor + 28, 2).unpack1("v")
+        { data_start: cursor + 30 + name_len + extra_len, extra_length: extra_len }
+      end
+    end
+
+    def validate_raw_preserve_entry!(source, central, entry)
+      raise render_error("archive_comment", "/archive/comment", "raw-preserving ZIP renderer does not yet preserve archive comments") if central[:archive_comment]
+      record = central[:records].fetch(entry.path)
+      raise render_error("encrypted_member", schema_path(entry.normalized_path), "raw-preserving ZIP renderer rejects encrypted member #{entry.normalized_path}") unless (record[:flags] & 0x1).zero?
+      raise render_error("unsupported_compression", schema_path(entry.normalized_path), "raw-preserving ZIP renderer rejects unsupported compression #{entry.compression.inspect}") unless [0, 8].include?(record[:method])
+      raise render_error("central_directory_extra_field", schema_path(entry.normalized_path), "raw-preserving ZIP renderer does not yet preserve central-directory extra fields for #{entry.normalized_path}") unless record[:extra_length].zero?
+      raise render_error("member_comment", schema_path(entry.normalized_path), "raw-preserving ZIP renderer does not yet preserve member comments for #{entry.normalized_path}") unless record[:comment_length].zero?
+      local_extra = source.byteslice(entry.local_header_range.start_byte + 28, 2).unpack1("v")
+      raise render_error("local_header_extra_field", schema_path(entry.normalized_path), "raw-preserving ZIP renderer does not yet preserve local extra fields for #{entry.normalized_path}") unless local_extra.zero?
+    end
+
+    def unsafe_entries(entries, records)
+      seen = {}
+      entries.flat_map do |entry|
+        list = []
+        list << TreeHaver::ZipUnsafeEntry.new(path: entry.path, normalized_path: entry.normalized_path, category: "path_traversal", reason: "entry escapes the archive root") if escapes_root?(entry.path)
+        list << TreeHaver::ZipUnsafeEntry.new(path: entry.path, normalized_path: entry.normalized_path, category: "duplicate_normalized_path", reason: "normalized path collides with an existing entry") if seen[entry.normalized_path] && seen[entry.normalized_path] != entry.path
+        list << TreeHaver::ZipUnsafeEntry.new(path: entry.path, normalized_path: entry.normalized_path, category: "encrypted_member", reason: "encrypted member cannot be rendered by the default provider") unless (records[entry.path][:flags] & 0x1).zero?
+        list << TreeHaver::ZipUnsafeEntry.new(path: entry.path, normalized_path: entry.normalized_path, category: "signing_sensitive_member", reason: "signature-bearing member mutation is not enabled") if signing_sensitive?(entry.normalized_path)
+        seen[entry.normalized_path] = entry.path
+        list
+      end
+    end
+
+    def rendered_local_record(entry, content, method, offset)
+      payload = method == 8 ? Zlib::Deflate.deflate(content) : content
+      crc = Zlib.crc32(content)
+      header = [LOCAL, 20, 0, method].pack("Vvvv") + DOS_EPOCH + [crc, payload.bytesize, content.bytesize, entry.path.bytesize, 0].pack("VVVvv") + entry.path
+      [header + payload, { name: entry.path, method: method, crc32: crc, compressed_size: payload.bytesize, uncompressed_size: content.bytesize, offset: offset, flags: 0 }]
+    end
+
+    def central_directory_record(record)
+      [CENTRAL, 20, 20, record[:flags], record[:method]].pack("Vvvvv") + DOS_EPOCH + [record[:crc32], record[:compressed_size], record[:uncompressed_size], record[:name].bytesize, 0, 0, 0, 0, 0, record[:offset]].pack("VVVvvvvvVV") + record[:name]
+    end
+
+    def eocd_record(entries, size, offset)
+      [EOCD, 0, 0, entries, entries, size, offset, 0].pack("VvvvvVVv")
+    end
+
+    def raw_local_record_ranges(source, entries)
+      ordered = entries.values.sort_by { |entry| entry.local_header_range.start_byte }
+      ordered.each_with_index.to_h do |entry, index|
+        end_byte = ordered[index + 1]&.local_header_range&.start_byte || entry.central_directory_range.start_byte
+        [entry.normalized_path, TreeHaver::ByteRange.new(start_byte: entry.local_header_range.start_byte, end_byte: end_byte)]
+      end
+    end
+
+    def central_record_from_entry(entry, offset)
+      { name: entry.path, method: entry.compression == "deflate" ? 8 : 0, crc32: entry.crc32.to_i(16), compressed_size: entry.compressed_size, uncompressed_size: entry.uncompressed_size, offset: offset, flags: 0 }
+    end
+
+    def path_entry(name, content)
+      TreeHaver::ZipArchiveEntry.new(path: name, normalized_path: normalize_zip_path(name), directory: name.end_with?("/"), compression: "stored", compressed_size: content.bytesize, uncompressed_size: content.bytesize, crc32: "%08x" % Zlib.crc32(content), local_header_range: TreeHaver::ByteRange.new(start_byte: 0, end_byte: 0), data_range: TreeHaver::ByteRange.new(start_byte: 0, end_byte: 0), central_directory_range: TreeHaver::ByteRange.new(start_byte: 0, end_byte: 0))
+    end
+
+    def decision(report, path, operation, disposition, reason)
+      report.member_decisions << TreeHaver::ZipMemberDecision.new(normalized_path: path, operation: operation, disposition: disposition, reason: reason)
+      report.merge_report.rewritten_nodes << schema_path(path)
+    end
+
+    def normalize_zip_path(path)
+      path.tr("\\", "/").split("/").each_with_object([]) { |part, stack| part == ".." ? stack.pop : stack << part unless part == "." }.join("/")
+    end
+
+    def compression_name(method)
+      case method
+      when 0 then "stored"
+      when 8 then "deflate"
+      else "method-#{method}"
+      end
+    end
+
+    def escapes_root?(path)
+      path.start_with?("/") || path.tr("\\", "/").split("/").then { |parts| depth = 0; parts.any? { |part| part == ".." ? (depth -= 1) : (depth += 1 unless part == "."); depth.negative? } }
+    end
+
+    def signing_sensitive?(path)
+      path.upcase.start_with?("META-INF/") && [".RSA", ".DSA", ".EC", ".SF"].any? { |suffix| path.upcase.end_with?(suffix) }
+    end
+
+    def same_entry?(left, right)
+      left && right && left.path == right.path && left.compression == right.compression && left.compressed_size == right.compressed_size && left.uncompressed_size == right.uncompressed_size && left.crc32 == right.crc32
+    end
+
+    def entries_by_path(entries)
+      entries.to_h { |entry| [entry.normalized_path, entry] }
+    end
+
+    def nested_family(path)
+      return "markdown" if path.match?(/\.m(?:d|arkdown)\z/i)
+      return "json" if path.end_with?(".json")
+      return "yaml" if path.match?(/\.ya?ml\z/i)
+      return "xml" if path.end_with?(".xml")
+    end
+
+    def schema_path(path)
+      "/entries/by_path/#{path}"
+    end
+
+    def diagnostic(category, schema_path, message)
+      TreeHaver::BinaryDiagnostic.new(severity: "error", category: category, message: message, schema_path: schema_path)
+    end
+
+    def render_error(category, schema_path, message)
+      RenderError.new(diagnostic(category, schema_path, message))
+    end
+  end
+end

data/lib/zip-merge.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "zip/merge"

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: zip-merge
+version: !ruby/object:Gem::Version
+  version: 7.0.0
+platform: ruby
+authors:
+- Peter H. Boling
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEgDCCAuigAwIBAgIBATANBgkqhkiG9w0BAQsFADBDMRUwEwYDVQQDDAxwZXRl
+  ci5ib2xpbmcxFTATBgoJkiaJk/IsZAEZFgVnbWFpbDETMBEGCgmSJomT8ixkARkW
+  A2NvbTAeFw0yNTA1MDQxNTMzMDlaFw00NTA0MjkxNTMzMDlaMEMxFTATBgNVBAMM
+  DHBldGVyLmJvbGluZzEVMBMGCgmSJomT8ixkARkWBWdtYWlsMRMwEQYKCZImiZPy
+  LGQBGRYDY29tMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAruUoo0WA
+  uoNuq6puKWYeRYiZekz/nsDeK5x/0IEirzcCEvaHr3Bmz7rjo1I6On3gGKmiZs61
+  LRmQ3oxy77ydmkGTXBjruJB+pQEn7UfLSgQ0xa1/X3kdBZt6RmabFlBxnHkoaGY5
+  mZuZ5+Z7walmv6sFD9ajhzj+oIgwWfnEHkXYTR8I6VLN7MRRKGMPoZ/yvOmxb2DN
+  coEEHWKO9CvgYpW7asIihl/9GMpKiRkcYPm9dGQzZc6uTwom1COfW0+ZOFrDVBuV
+  FMQRPswZcY4Wlq0uEBLPU7hxnCL9nKK6Y9IhdDcz1mY6HZ91WImNslOSI0S8hRpj
+  yGOWxQIhBT3fqCBlRIqFQBudrnD9jSNpSGsFvbEijd5ns7Z9ZMehXkXDycpGAUj1
+  to/5cuTWWw1JqUWrKJYoifnVhtE1o1DZ+LkPtWxHtz5kjDG/zR3MG0Ula0UOavlD
+  qbnbcXPBnwXtTFeZ3C+yrWpE4pGnl3yGkZj9SMTlo9qnTMiPmuWKQDatAgMBAAGj
+  fzB9MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQWBBQE8uWvNbPVNRXZ
+  HlgPbc2PCzC4bjAhBgNVHREEGjAYgRZwZXRlci5ib2xpbmdAZ21haWwuY29tMCEG
+  A1UdEgQaMBiBFnBldGVyLmJvbGluZ0BnbWFpbC5jb20wDQYJKoZIhvcNAQELBQAD
+  ggGBAJbnUwfJQFPkBgH9cL7hoBfRtmWiCvdqdjeTmi04u8zVNCUox0A4gT982DE9
+  wmuN12LpdajxZONqbXuzZvc+nb0StFwmFYZG6iDwaf4BPywm2e/Vmq0YG45vZXGR
+  L8yMDSK1cQXjmA+ZBKOHKWavxP6Vp7lWvjAhz8RFwqF9GuNIdhv9NpnCAWcMZtpm
+  GUPyIWw/Cw/2wZp74QzZj6Npx+LdXoLTF1HMSJXZ7/pkxLCsB8m4EFVdb/IrW/0k
+  kNSfjtAfBHO8nLGuqQZVH9IBD1i9K6aSs7pT6TW8itXUIlkIUI2tg5YzW6OFfPzq
+  QekSkX3lZfY+HTSp/o+YvKkqWLUV7PQ7xh1ZYDtocpaHwgxe/j3bBqHE+CUPH2vA
+  0V/FwdTRWcwsjVoOJTrYcff8pBZ8r2MvtAc54xfnnhGFzeRHfcltobgFxkAXdE6p
+  DVjBtqT23eugOqQ73umLcYDZkc36vnqGxUBSsXrzY9pzV5gGr2I8YUxMqf6ATrZt
+  L9nRqA==
+  -----END CERTIFICATE-----
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tree_haver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+description: Portable ZIP inventory, planning, nested dispatch, and raw-preservation
+  rendering helpers for Structured Merge.
+email:
+- info@structuredmerge.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/zip-merge.rb
+- lib/zip/merge.rb
+- lib/zip/merge/version.rb
+homepage: https://github.com/structuredmerge/structuredmerge-ruby
+licenses:
+- AGPL-3.0-only
+- PolyForm-Small-Business-1.0.0
+metadata:
+  homepage_uri: https://structuredmerge.org
+  source_code_uri: https://github.com/structuredmerge/structuredmerge-ruby/tree/v7.0.0
+  changelog_uri: https://github.com/structuredmerge/structuredmerge-ruby/blob/v7.0.0/CHANGELOG.md
+  bug_tracker_uri: https://github.com/structuredmerge/structuredmerge-ruby/issues
+  documentation_uri: https://www.rubydoc.info/gems/zip-merge/7.0.0
+  funding_uri: https://github.com/sponsors/pboling
+  wiki_uri: https://github.com/structuredmerge/structuredmerge-ruby/wiki
+  discord_uri: https://discord.gg/3qme4XHNKN
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 4.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.10
+specification_version: 4
+summary: Structured Merge ZIP merge planning and rendering helpers for Ruby
+test_files: []