pq_crypto 0.4.2 → 0.5.1

data/script/vendor_libs.rb

@@ -3,172 +3,107 @@
 
 require "digest"
 require "fileutils"
-require "open-uri"
-require "rubygems/package"
+require "open3"
+require "optparse"
 require "tmpdir"
-require "zlib"
 
 VENDOR_DIR = File.expand_path("../ext/pqcrypto/vendor", __dir__)
 MANIFEST_PATH = File.join(VENDOR_DIR, ".vendored")
 
-DEFAULT_PQCLEAN = {
-  version: "2cc64716044832eea747234ddbffc06746ab815d",
-  url: "https://github.com/PQClean/PQClean/archive/2cc64716044832eea747234ddbffc06746ab815d.tar.gz",
-  strip: "PQClean-2cc64716044832eea747234ddbffc06746ab815d",
-  sha256: "0e92076a79082a8d220e27227f37b280fb2ce050af412babd2bc755ab37b871a"
+PINS = {
+  mlkem: {
+    repo: "https://github.com/pq-code-package/mlkem-native.git",
+    ref: "v1.1.0",
+    commit: "d2cae2be522a67bfae26100fdb520576f1b2ef90",
+    tree_sha256: "c225de87a69e6d6360cddc4b5839b03e65fa9d5a1112a5f19700c905b7e74512",
+    target: "mlkem-native",
+    source_dir: "mlkem"
+  },
+  mldsa: {
+    repo: "https://github.com/pq-code-package/mldsa-native.git",
+    ref: "v1.0.0-beta",
+    commit: "db65535319d9750d75d34c6d170677415f9d2c46",
+    tree_sha256: "3b2cb648dade4540191f08d606b422042bf781fb37b434934ab02b58a0121f5c",
+    target: "mldsa-native",
+    source_dir: "mldsa"
+  }
 }.freeze
 
-KEEP_DIRS = %w[
-  crypto_kem/ml-kem-512/clean
-  crypto_kem/ml-kem-768/clean
-  crypto_kem/ml-kem-1024/clean
-  crypto_sign/ml-dsa-44/clean
-  crypto_sign/ml-dsa-65/clean
-  crypto_sign/ml-dsa-87/clean
-  common
-].freeze
-
-WARNING = <<~TEXT.freeze
-  WARNING: this script is a manual vendor refresh tool.
-
-  pq_crypto relies on the vendored PQClean snapshot committed to the repository.
-  Running this script will replace ext/pqcrypto/vendor with a fresh upstream copy.
-  Use the pinned defaults unless you are intentionally updating the upstream snapshot.
-TEXT
-
-def load_manifest(path)
-  return {} unless File.exist?(path)
-
-  File.readlines(path, chomp: true).each_with_object({}) do |line, acc|
-    next if line.strip.empty? || line.lstrip.start_with?("#")
-    key, value = line.split("=", 2)
-    next if key.nil? || value.nil?
-
-    acc[key] = value
-  end
-end
+VENDORED_DOCS = %w[LICENSE LICENSE.txt README.md SECURITY.md BUILDING.md RELEASE.md META.yml].freeze
+NORMALIZED_MTIME = Time.utc(2000, 1, 1).freeze
+MANIFEST_HEADER = "# pq_crypto vendor manifest. Do not edit by hand. Regenerate with: ruby script/vendor_libs.rb"
 
-def manifest_value(manifest, key)
-  value = manifest[key]
-  return nil if value.nil? || value.strip.empty? || value == "unrecorded"
-
-  value
-end
+options = { mode: :sync }
+OptionParser.new do |opts|
+  opts.banner = "Usage: vendor_libs.rb [--verify | --sync | --bump]"
+  opts.on("--verify", "Verify existing vendor tree against pinned tree_sha256 (no network)") { options[:mode] = :verify }
+  opts.on("--sync", "Re-clone at pinned commits and rebuild vendor tree (idempotent)") { options[:mode] = :sync }
+  opts.on("--bump", "Re-clone and print new tree_sha256 values to update PINS in this file") { options[:mode] = :bump }
+end.parse!
 
-def commit_archive_url?(url)
-  url.match?(%r{\Ahttps://(?:github\.com|codeload\.github\.com)/PQClean/PQClean/(?:archive|tar\.gz)/[0-9a-f]{40}(?:\.tar\.gz)?\z}i) ||
-    url.match?(%r{\Ahttps://github\.com/PQClean/PQClean/archive/[0-9a-f]{40}\.tar\.gz\z}i)
+def sh!(cmd)
+  system(*cmd) || abort("command failed: #{cmd.join(" ")}")
 end
 
-def build_vendor_config
-  manifest = load_manifest(MANIFEST_PATH)
-
-  version = ENV["PQCLEAN_VERSION"] || manifest_value(manifest, "pqclean_version") || DEFAULT_PQCLEAN[:version]
-  url = ENV["PQCLEAN_URL"] || manifest_value(manifest, "pqclean_url") || DEFAULT_PQCLEAN[:url]
-  sha256 = ENV["PQCLEAN_SHA256"] || manifest_value(manifest, "pqclean_archive_sha256") || DEFAULT_PQCLEAN[:sha256]
-  strip = ENV["PQCLEAN_STRIP"] || manifest_value(manifest, "pqclean_strip") || DEFAULT_PQCLEAN[:strip]
-
-  {
-    version: version,
-    url: url,
-    sha256: sha256,
-    strip: strip,
-    keep: KEEP_DIRS
-  }.freeze
+def capture!(*cmd)
+  out, status = Open3.capture2(*cmd)
+  abort("command failed: #{cmd.join(" ")}") unless status.success?
+  out.strip
 end
 
-def validate_vendor_config!(config)
-  required = %i[version url strip]
-  missing = required.select { |key| config[key].to_s.strip.empty? }
-  return if missing.empty?
-
-  abort <<~MSG
-    Missing required vendoring configuration: #{missing.join(", ")}
-
-    Example:
-      PQCLEAN_VERSION=<full-git-commit> \
-      PQCLEAN_URL=https://github.com/PQClean/PQClean/archive/<full-git-commit>.tar.gz \
-      PQCLEAN_STRIP=PQClean-<full-git-commit> \
-      PQCLEAN_SHA256=<archive-sha256> \
-      bundle exec ruby script/vendor_libs.rb
-  MSG
+def vendor_candidate?(path)
+  return false if File.symlink?(path)
+  return false unless File.file?(path)
+  return false if path.split(File::SEPARATOR).any? { |seg| seg.start_with?(".") && seg != "." && seg != ".." }
+  true
 end
 
-def validate_pinning!(config)
-  if config[:sha256].to_s.strip.empty?
-    abort <<~MSG
-      Refusing to vendor without PQCLEAN_SHA256.
-
-      Use the built-in pinned defaults, or provide all of:
-        PQCLEAN_VERSION
-        PQCLEAN_URL
-        PQCLEAN_STRIP
-        PQCLEAN_SHA256
-    MSG
+def normalize_tree!(directory)
+  Dir.glob(File.join(directory, "**", "*"), File::FNM_DOTMATCH).each do |path|
+    base = File.basename(path)
+    next if base == "." || base == ".."
+    next if File.symlink?(path)
+    if File.file?(path)
+      File.chmod(0o644, path)
+      File.utime(NORMALIZED_MTIME, NORMALIZED_MTIME, path)
+    elsif File.directory?(path)
+      File.chmod(0o755, path)
+    end
   end
-
-  return if commit_archive_url?(config[:url])
-
-  abort <<~MSG
-    Refusing to vendor from a non-commit archive URL.
-
-    Use a content-addressed full commit archive URL from the PQClean repository.
-  MSG
-end
-
-def download(url, destination)
-  puts "Downloading #{url}"
-  URI.open(url) { |remote| File.binwrite(destination, remote.read) }
+  File.utime(NORMALIZED_MTIME, NORMALIZED_MTIME, directory) if File.directory?(directory)
 end
 
-def verify_checksum!(archive, expected_sha256)
-  actual = Digest::SHA256.file(archive).hexdigest
-  abort "SHA256 mismatch: expected #{expected_sha256}, got #{actual}" unless actual == expected_sha256
-  actual
-end
+def copy_sources!(source_root, target_root, source_dir)
+  required = File.join(source_root, source_dir)
+  abort "missing required upstream directory: #{required}" unless Dir.exist?(required)
 
-def extract_subset(archive, destination, strip_prefix:, keep_dirs:)
-  prefix_re = /\A#{Regexp.escape(strip_prefix)}\//
+  FileUtils.rm_rf(File.join(target_root, source_dir))
+  FileUtils.mkdir_p(File.join(target_root, source_dir))
 
-  Gem::Package::TarReader.new(Zlib::GzipReader.open(archive)) do |tar|
-    tar.each do |entry|
-      relative_path = entry.full_name.sub(prefix_re, "")
-      next if relative_path.empty? || relative_path == entry.full_name
-      next unless keep_dirs.any? { |dir| relative_path.start_with?(dir) }
-
-      target = File.join(destination, relative_path)
-
-      if entry.directory?
-        FileUtils.mkdir_p(target)
-      elsif entry.file?
-        FileUtils.mkdir_p(File.dirname(target))
-        File.binwrite(target, entry.read)
-      end
-    end
+  Dir.glob(File.join(required, "**", "*"), File::FNM_DOTMATCH).sort.each do |path|
+    next unless vendor_candidate?(path)
+    relative = path.sub(/\A#{Regexp.escape(required)}\//, "")
+    dest = File.join(target_root, source_dir, relative)
+    FileUtils.mkdir_p(File.dirname(dest))
+    FileUtils.cp(path, dest, preserve: false)
   end
-end
 
-def write_manifest!(config:, archive_sha256:, tree_sha256:)
-  File.write(
-    MANIFEST_PATH,
-    <<~TEXT
-      pqclean_version=#{config[:version]}
-      pqclean_url=#{config[:url]}
-      pqclean_archive_sha256=#{archive_sha256}
-      pqclean_strip=#{config[:strip]}
-      pqclean_tree_sha256=#{tree_sha256}
-    TEXT
-  )
+  VENDORED_DOCS.each do |relative|
+    src = File.join(source_root, relative)
+    next if File.symlink?(src)
+    next unless File.file?(src)
+    FileUtils.cp(src, File.join(target_root, relative), preserve: false)
+  end
 end
 
 def tree_sha256_for(directory)
   entries = Dir.glob(File.join(directory, "**", "*"), File::FNM_DOTMATCH)
-               .reject { |path| File.directory?(path) }
+               .reject { |p| File.directory?(p) || File.symlink?(p) || %w[. ..].include?(File.basename(p)) }
                .sort
 
   digest = Digest::SHA256.new
   entries.each do |path|
-    relative = path.delete_prefix("#{directory}/")
+    relative = path.sub(/\A#{Regexp.escape(directory)}\/?/, "")
     digest << relative << "\0"
     digest << File.binread(path)
     digest << "\0"
@@ -176,27 +111,166 @@ def tree_sha256_for(directory)
   digest.hexdigest
 end
 
-config = build_vendor_config
-validate_vendor_config!(config)
-validate_pinning!(config)
+def vendor_one(name, pin)
+  target = File.join(VENDOR_DIR, pin[:target])
+  FileUtils.rm_rf(target)
+  FileUtils.mkdir_p(target)
+
+  Dir.mktmpdir("pqcrypto-#{name}-") do |tmpdir|
+    clone_dir = File.join(tmpdir, pin[:target])
+    sh!(["git", "clone", "--depth", "1", "--branch", pin[:ref], pin[:repo], clone_dir])
+    actual_commit = capture!("git", "-C", clone_dir, "rev-parse", "HEAD")
+
+    if actual_commit != pin[:commit]
+      sh!(["git", "-C", clone_dir, "fetch", "--depth", "1", "origin", pin[:commit]])
+      sh!(["git", "-C", clone_dir, "checkout", "--detach", pin[:commit]])
+      actual_commit = capture!("git", "-C", clone_dir, "rev-parse", "HEAD")
+    end
+
+    abort "commit mismatch for #{name}: expected #{pin[:commit]}, got #{actual_commit}" unless actual_commit == pin[:commit]
+
+    copy_sources!(clone_dir, target, pin[:source_dir])
+  end
 
-puts WARNING
-puts "Vendoring PQClean into #{VENDOR_DIR}"
-puts "Pinned ref: #{config[:version]}"
-puts "Archive checksum: #{config[:sha256]}"
+  normalize_tree!(target)
+  tree_sha256_for(target)
+end
 
-FileUtils.rm_rf(VENDOR_DIR)
-FileUtils.mkdir_p(VENDOR_DIR)
+def manifest_body_lines(results)
+  lines = ["backend=PQ Code Package native only", "pqclean=removed"]
+  results.each do |name, data|
+    prefix = "#{name}_native"
+    lines << "#{prefix}_repo=#{data[:repo]}"
+    lines << "#{prefix}_ref=#{data[:ref]}"
+    lines << "#{prefix}_commit=#{data[:commit]}"
+    lines << "#{prefix}_tree_sha256=#{data[:tree_sha256]}"
+  end
+  lines
+end
 
-Dir.mktmpdir("pq_crypto-vendor") do |tmpdir|
-  archive = File.join(tmpdir, "pqclean.tar.gz")
-  destination = File.join(VENDOR_DIR, "pqclean")
+def manifest_signature(body_lines)
+  Digest::SHA256.hexdigest(body_lines.join("\n") + "\n")
+end
+
+def write_manifest(results)
+  body = manifest_body_lines(results)
+  sig = manifest_signature(body)
+  content = ([MANIFEST_HEADER] + body + ["manifest_sha256=#{sig}"]).join("\n") + "\n"
+  File.write(MANIFEST_PATH, content)
+  File.chmod(0o644, MANIFEST_PATH)
+  File.utime(NORMALIZED_MTIME, NORMALIZED_MTIME, MANIFEST_PATH)
+end
 
-  download(config[:url], archive)
-  archive_sha256 = verify_checksum!(archive, config[:sha256])
-  extract_subset(archive, destination, strip_prefix: config[:strip], keep_dirs: config[:keep])
-  write_manifest!(config: config, archive_sha256: archive_sha256, tree_sha256: tree_sha256_for(destination))
+def parse_manifest(path)
+  return { kv: {}, body: [], signature: nil } unless File.exist?(path)
+  body = []
+  kv = {}
+  signature = nil
+  File.readlines(path, chomp: true).each do |line|
+    next if line.start_with?("#")
+    next if line.empty?
+    if line.start_with?("manifest_sha256=")
+      signature = line.split("=", 2).last
+    else
+      body << line
+      k, v = line.split("=", 2)
+      kv[k] = v if k && v
+    end
+  end
+  { kv: kv, body: body, signature: signature }
 end
 
-puts "Done. PQClean sources are now available in ext/pqcrypto/vendor/."
-puts "Next step: review vendor diffs, then bundle exec rake compile"
+case options[:mode]
+when :verify
+  manifest = parse_manifest(MANIFEST_PATH)
+  failures = []
+
+  if manifest[:signature].nil?
+    failures << "manifest: missing manifest_sha256 line"
+  else
+    expected_sig = manifest_signature(manifest[:body])
+    failures << "manifest: signature mismatch (manifest_sha256=#{manifest[:signature]}, computed=#{expected_sig})" if expected_sig != manifest[:signature]
+  end
+
+  PINS.each do |name, pin|
+    target = File.join(VENDOR_DIR, pin[:target])
+    unless Dir.exist?(target)
+      failures << "#{name}: vendor directory missing (#{target})"
+      next
+    end
+
+    manifest_commit = manifest[:kv]["#{name}_native_commit"]
+    if manifest_commit != pin[:commit]
+      failures << "#{name}: manifest commit (#{manifest_commit.inspect}) != PINS commit (#{pin[:commit]})"
+    end
+
+    manifest_tree = manifest[:kv]["#{name}_native_tree_sha256"]
+    if manifest_tree != pin[:tree_sha256]
+      failures << "#{name}: manifest tree_sha256 (#{manifest_tree.inspect}) != PINS tree_sha256 (#{pin[:tree_sha256]})"
+    end
+
+    actual_tree = tree_sha256_for(target)
+    if actual_tree != pin[:tree_sha256]
+      failures << "#{name}: filesystem tree_sha256 (#{actual_tree}) != PINS tree_sha256 (#{pin[:tree_sha256]})"
+    end
+  end
+
+  if failures.empty?
+    puts "vendor verify: ok"
+    exit 0
+  else
+    failures.each { |f| warn f }
+    exit 1
+  end
+
+when :sync
+  if Dir.exist?(VENDOR_DIR) && File.exist?(MANIFEST_PATH)
+    manifest = parse_manifest(MANIFEST_PATH)
+    sig_ok = manifest[:signature] && manifest_signature(manifest[:body]) == manifest[:signature]
+    pins_ok = sig_ok && PINS.all? do |name, pin|
+      target = File.join(VENDOR_DIR, pin[:target])
+      Dir.exist?(target) &&
+        manifest[:kv]["#{name}_native_commit"] == pin[:commit] &&
+        manifest[:kv]["#{name}_native_tree_sha256"] == pin[:tree_sha256] &&
+        tree_sha256_for(target) == pin[:tree_sha256]
+    end
+
+    if pins_ok
+      puts "vendor already at pinned commits and tree_sha256; nothing to do"
+      exit 0
+    end
+  end
+
+  FileUtils.rm_rf(VENDOR_DIR)
+  FileUtils.mkdir_p(VENDOR_DIR)
+
+  results = {}
+  PINS.each do |name, pin|
+    actual_tree = vendor_one(name, pin)
+    if actual_tree != pin[:tree_sha256]
+      abort "#{name}: tree_sha256 drift (PINS=#{pin[:tree_sha256]}, actual=#{actual_tree}). " \
+            "Upstream changed under the pinned commit, or the vendor algorithm changed. " \
+            "If intentional, run with --bump to print new pins."
+    end
+    results[name] = pin.merge(tree_sha256: actual_tree)
+  end
+  write_manifest(results)
+  puts "vendor sync: ok"
+  results.each { |name, data| puts "  #{name}: commit=#{data[:commit]} tree_sha256=#{data[:tree_sha256]}" }
+
+when :bump
+  FileUtils.rm_rf(VENDOR_DIR)
+  FileUtils.mkdir_p(VENDOR_DIR)
+
+  results = {}
+  PINS.each do |name, pin|
+    actual_tree = vendor_one(name, pin)
+    results[name] = pin.merge(tree_sha256: actual_tree)
+  end
+  write_manifest(results)
+
+  puts "Update PINS in script/vendor_libs.rb to:"
+  results.each do |name, data|
+    puts "  PINS[:#{name}][:tree_sha256] = #{data[:tree_sha256].inspect}"
+  end
+end