ruby-magic 0.3.0 → 0.5.0

data/ext/magic/extconf.rb

@@ -1,42 +1,195 @@
 # frozen_string_literal: true
 
+require 'find'
 require 'mkmf'
-require 'digest'
-require 'open-uri'
-
-LIBMAGIC_TAG = '5.39'.freeze
-
-workdir = Dir.pwd
-libdir = File.join(workdir, 'file-' + LIBMAGIC_TAG)
-gemdir = File.expand_path(File.join(__dir__, '../..'))
-gem_ext_dir = File.join(gemdir, 'lib', 'ext')
-gem_include_dir = File.join(gem_ext_dir, 'include')
-gem_lib_dir = File.join(gem_ext_dir, 'lib')
-build_lib_dir = File.join(libdir, 'src', '.libs')
-
-expected_sha256 = 'f05d286a76d9556243d0cb05814929c2ecf3a5ba07963f8f70bfaaa70517fad1'
-filename = "#{workdir}/file.tar.gz"
-
-unless File.exist?(filename)
-  File.open(filename, 'wb') do |target_file|
-    URI.open("https://fossies.org/linux/misc/file-#{LIBMAGIC_TAG}.tar.gz", 'rb') do |read_file|
-      target_file.write(read_file.read)
+require 'pathname'
+
+# helpful constants
+PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+
+# The gem version constraint in the Rakefile is not respected at install time.
+# Keep this version in sync with the one in the Rakefile !
+REQUIRED_MINI_PORTILE_VERSION = "~> 2.6"
+
+MAGIC_HELP_MESSAGE = <<~HELP
+  USAGE: ruby #{$0} [options]
+
+    Flags that are always valid:
+
+      --use-system-libraries
+      --enable-system-libraries
+          Use system libraries instead of building and using the packaged libraries.
+
+      --disable-system-libraries
+          Use the packaged libraries, and ignore the system libraries. This is the default on most
+          platforms, and overrides `--use-system-libraries` and the environment variable
+          `MAGIC_USE_SYSTEM_LIBRARIES`.
+
+      --disable-clean
+          Do not clean out intermediate files after successful build.
+
+    Flags only used when building and using the packaged libraries:
+
+      --disable-static
+          Do not statically link packaged libraries, instead use shared libraries.
+
+      --enable-cross-build
+          Enable cross-build mode. (You probably do not want to set this manually.)
+
+    Flags only used when using system libraries:
+
+      Related to libmagic:
+
+        --with-magic-dir=DIRECTORY
+            Look for libmagic headers and library in DIRECTORY.
+
+        --with-magic-lib=DIRECTORY
+            Look for libmagic library in DIRECTORY.
+
+        --with-magic-include=DIRECTORY
+            Look for libmagic headers in DIRECTORY.
+
+    Environment variables used:
+
+      CC
+          Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']`
+
+      CPPFLAGS
+          If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor
+
+      CFLAGS
+          If this string is accepted by the compiler, add it to the flags passed to the compiler
+
+      LDFLAGS
+          If this string is accepted by the linker, add it to the flags passed to the linker
+
+      LIBS
+          Add this string to the flags passed to the linker
+HELP
+
+def process_recipe(name, version, static_p, cross_p)
+  require 'rubygems'
+  gem('mini_portile2', REQUIRED_MINI_PORTILE_VERSION)
+  require 'mini_portile2'
+  message("Using mini_portile version #{MiniPortile::VERSION}\n")
+
+  MiniPortile.new(name, version).tap do |recipe|
+    # Prefer host_alias over host in order to use i586-mingw32msvc as
+    # correct compiler prefix for cross build, but use host if not set.
+    recipe.host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"]
+    recipe.target = File.join(PACKAGE_ROOT_DIR, "ports")
+    recipe.patch_files = Dir[File.join(PACKAGE_ROOT_DIR, "patches", name, "*.patch")].sort
+    recipe.configure_options << "--libdir=#{File.join(recipe.path, 'lib')}"
+
+    yield recipe
+
+    env = Hash.new do |hash, key|
+      hash[key] = (ENV[key]).to_s
     end
-  end
 
-  checksum = Digest::SHA256.hexdigest(File.read(filename))
+    recipe.configure_options.flatten!
+
+    recipe.configure_options.delete_if do |option|
+      case option
+      when /\A(\w+)=(.*)\z/
+        env[Regexp.last_match(1)] = if env.key?(Regexp.last_match(1))
+          concat_flags(env[Regexp.last_match(1)], Regexp.last_match(2))
+        else
+          Regexp.last_match(2)
+        end
+        true
+      else
+        false
+      end
+    end
+
+    recipe.configure_options = [
+      "--disable-silent-rules",
+      "--disable-dependency-tracking",
+      "--enable-fsect-man5"
+    ]
+
+    if static_p
+      recipe.configure_options += [
+        "--disable-shared",
+        "--enable-static",
+      ]
+      env["CFLAGS"] = concat_flags(env["CFLAGS"], "-fPIC")
+    else
+      recipe.configure_options += [
+        "--enable-shared",
+        "--disable-static",
+      ]
+    end
+
+    if cross_p
+      recipe.configure_options += [
+        "--target=#{recipe.host}",
+        "--host=#{recipe.host}",
+      ]
+    end
+
+    if RbConfig::CONFIG['target_cpu'] == 'universal'
+      %w[CFLAGS LDFLAGS].each do |key|
+        unless env[key].include?('-arch')
+          env[key] = concat_flags(env[key], RbConfig::CONFIG['ARCH_FLAG'])
+        end
+      end
+    end
+
+    recipe.configure_options += env.map do |key, value|
+      "#{key}=#{value.strip}"
+    end
+
+    checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed"
+
+    if File.exist?(checkpoint)
+      message("Building Ruby Magic with a packaged version of #{name}-#{version}.\n")
+    else
+      message(<<~EOM)
+        ---------- IMPORTANT NOTICE ----------
+        Building Ruby Magic with a packaged version of #{name}-#{version}.
+        Configuration options: #{recipe.configure_options.shelljoin}
+      EOM
 
-  if checksum != expected_sha256
-    raise "SHA256 of #{filename} does not match: got #{checksum}, expected #{expected_sha256}"
+      unless recipe.patch_files.empty?
+        message("The following patches are being applied:\n")
+
+        recipe.patch_files.each do |patch|
+          message("  - %s\n" % File.basename(patch))
+        end
+      end
+
+      recipe.cook
+
+      FileUtils.touch(checkpoint)
+    end
+
+    recipe.activate
   end
 end
 
-system("tar -xzf #{filename}") || raise('ERROR')
-system("cd #{libdir} && ./configure --prefix=#{gem_ext_dir} && make install") || raise('ERROR')
+#
+#  utility functions
+#
+def config_clean?
+  enable_config('clean', true)
+end
+
+def config_static?
+  default_static = !truffle?
+  enable_config("static", default_static)
+end
+
+def config_cross_build?
+  enable_config("cross-build")
+end
 
-$LOCAL_LIBS << '-lmagic'
-$LIBPATH << gem_lib_dir
-$CFLAGS << " -I #{libdir}/src"
+def config_system_libraries?
+  enable_config("system-libraries", ENV.key?("MAGIC_USE_SYSTEM_LIBRARIES")) do |_, default|
+    arg_config('--use-system-libraries', default)
+  end
+end
 
 def darwin?
   RbConfig::CONFIG['target_os'] =~ /darwin/
@@ -46,38 +199,144 @@ def windows?
   RbConfig::CONFIG['target_os'] =~ /mswin|mingw32|windows/
 end
 
+def truffle?
+  ::RUBY_ENGINE == 'truffleruby'
+end
+
+def concat_flags(*args)
+  args.compact.join(" ")
+end
+
+def do_help
+  print(MAGIC_HELP_MESSAGE)
+  exit!(0)
+end
+
+def do_clean
+  root = Pathname(PACKAGE_ROOT_DIR)
+  pwd  = Pathname(Dir.pwd)
+
+  # Skip if this is a development work tree
+  unless (root + '.git').exist?
+    message("Cleaning files only used during build.\n")
+
+    # (root + 'tmp') cannot be removed at this stage because
+    # libmagic.so is yet to be copied to lib.
+
+    # clean the ports build directory
+    Pathname.glob(pwd.join('tmp', '*', 'ports')) do |dir|
+      FileUtils.rm_rf(dir, verbose: true)
+    end
+
+    FileUtils.rm_rf(root + 'ports' + 'archives', verbose: true)
+
+    # Remove everything but share/ directory
+    remove_paths = %w[bin include]
+    remove_paths << 'lib' if config_static?
+
+    Pathname.glob(File.join(root, 'ports', '*', 'libmagic', '*')) do |dir|
+      remove_paths.each do |path|
+        remove_dir = File.join(dir, path)
+        FileUtils.rm_rf(remove_dir, verbose: true)
+      end
+    end
+  end
+
+  exit!(0)
+end
+
+#
+#  main
+#
+do_help if arg_config('--help')
+do_clean if arg_config('--clean')
+
 if ENV['CC']
   RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC']
 end
 
 ENV['CC'] = RbConfig::CONFIG['CC']
 
-$CFLAGS += ' -std=c99 -fPIC'
-$CFLAGS += ' -Wall -Wextra -pedantic'
+append_cflags(ENV["CFLAGS"].split) if ENV["CFLAGS"]
+append_cppflags(ENV["CPPFLAGS"].split) if ENV["CPPFLAGS"]
+append_ldflags(ENV["LDFLAGS"].split) if ENV["LDFLAGS"]
+
+$LIBS = concat_flags($LIBS, ENV["LIBS"]) if ENV["LIBS"]
+
+if truffle?
+  $CPPFLAGS += " -DMAGIC_CUSTOM_CHECK_TYPE"
+end
+
+if config_system_libraries?
+  message "Building ruby-magic using system libraries.\n"
+
+  $CPPFLAGS += " -DMAGIC_SYSTEM_LIBRARIES"
+
+  dir_config('magic')
+else
+  message "Building ruby-magic using packaged libraries.\n"
+
+  require 'yaml'
+  dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, "dependencies.yml"))
+
+  static_p = config_static?
+  message "Static linking is #{static_p ? 'enabled' : 'disabled'}.\n"
+  cross_build_p = config_cross_build?
+  message "Cross build is #{cross_build_p ? 'enabled' : 'disabled'}.\n"
+
+  libmagic_recipe = process_recipe('libmagic', dependencies["libmagic"]["version"], static_p, cross_build_p) do |recipe|
+    recipe.files = [{
+      url: "https://ruby-magic.s3.nl-ams.scw.cloud/file-#{recipe.version}.tar.gz",
+      sha256: dependencies["libmagic"]["sha256"],
+    }]
+  end
+
+  $LIBPATH = [File.join(libmagic_recipe.path, 'lib')]
+  $CFLAGS << " -I#{File.join(libmagic_recipe.path, 'include')} "
+  $LDFLAGS += " -Wl,-rpath,#{libmagic_recipe.path}/lib"
+
+  if static_p
+    ENV['PKG_CONFIG_PATH'] = "#{libmagic_recipe.path}/lib/pkgconfig"
+    # mkmf appends -- to the first option
+    $LIBS += " " + pkg_config('libmagic', 'libs --static')
+    $LIBS += " " + File.join(libmagic_recipe.path, 'lib', "libmagic.#{$LIBEXT}")
+  end
+
+  if cross_build_p
+    # database files will be packaged up by the cross-compiling callback in the ExtensionTask
+    to_path = File.join(PACKAGE_ROOT_DIR, "ext/magic/share")
+    FileUtils.rm_rf(to_path, secure: true)
+    FileUtils.mkdir(to_path)
+    FileUtils.cp_r(Dir[File.join(libmagic_recipe.path, 'share/misc/*.mgc')], to_path)
+  end
+end
+
+$CFLAGS += ' -std=c99'
 
 if RbConfig::CONFIG['CC'] =~ /gcc/
   $CFLAGS += ' -O3' unless $CFLAGS =~ /-O\d/
-  $CFLAGS += ' -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline'
+end
+
+%w[
+  -Wcast-qual
+  -Wwrite-strings
+  -Wconversion
+  -Wmissing-noreturn
+  -Winline
+].select do |flag|
+  try_link('int main(void) { return 0; }', flag)
+end.each do |flag|
+  $CFLAGS += " " + flag
 end
 
 unless darwin?
-  $LDFLAGS += ' -Wl,--as-needed -Wl,--no-undefined'
+  $LDFLAGS += ' -Wl,--as-needed -Wl,--exclude-libs,ALL'
 end
 
 if windows?
   $LDFLAGS += ' -static-libgcc'
 end
 
-%w[
-  CFLAGS
-  CXXFLAGS
-  CPPFLAGS
-].each do |variable|
-  $CFLAGS += format(' %s', ENV[variable]) if ENV[variable]
-end
-
-$LDFLAGS += format(' %s', ENV['LDFLAGS']) if ENV['LDFLAGS']
-
 unless have_header('ruby.h')
   abort "\n" + (<<-EOS).gsub(/^[ ]{,3}/, '') + "\n"
     You appear to be missing Ruby development libraries and/or header
@@ -120,6 +379,13 @@ unless have_header('ruby.h')
   EOS
 end
 
+# these are needed for `rb_thread_call_without_gvl` to be properly detected on some linux systems.
+# specifically, rake-compiler-dock's redhat-based image needs these.
+have_library('pthread')
+have_library('rt')
+have_library('dl')
+have_library('crypt')
+
 have_func('rb_thread_call_without_gvl')
 have_func('rb_thread_blocking_region')
 
@@ -188,7 +454,17 @@ end
   have_func(f)
 end
 
-dir_config('magic', [gem_include_dir], [gem_lib_dir])
-
 create_header
 create_makefile('magic/magic')
+
+if config_clean?
+  # Do not clean if run in a development work tree.
+  File.open('Makefile', 'at') do |mk|
+    mk.print(<<~EOF)
+
+      all: clean-ports
+      clean-ports: $(DLLIB)
+      \t-$(Q)$(RUBY) $(srcdir)/extconf.rb --clean --#{static_p ? 'enable' : 'disable'}-static
+    EOF
+  end
+end

data/ext/magic/functions.c

@@ -4,14 +4,14 @@ extern "C" {
 
 #include "functions.h"
 
-int check_fd(int fd);
+static int check_fd(int fd);
 static int safe_dup(int fd);
 static int safe_close(int fd);
 static int safe_cloexec(int fd);
-int override_error_output(void *data);
-int restore_error_output(void *data);
+static int override_error_output(void *data);
+static int restore_error_output(void *data);
 
-inline int
+static inline int
 check_fd(int fd)
 {
 	errno = 0;
@@ -42,6 +42,7 @@ safe_dup(int fd)
 			goto error;
 		}
 	}
+
 	if (safe_cloexec(new_fd) < 0) {
 		local_errno = errno;
 		goto error;
@@ -78,6 +79,7 @@ safe_cloexec(int fd)
 		local_errno = errno;
 		goto error;
 	}
+
 	if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) {
 		local_errno = errno;
 		goto error;
@@ -89,18 +91,18 @@ error:
 	return -1;
 }
 
-int
+static int
 override_error_output(void *data)
 {
 	int local_errno;
-	mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
+	int flags = O_WRONLY | O_APPEND;
 	save_t *s = data;
 
 #if defined(HAVE_O_CLOEXEC)
-	mode |= O_CLOEXEC;
+	flags |= O_CLOEXEC;
 #endif
 
-	assert(s != NULL && \
+	assert(s != NULL &&
 	       "Must be a valid pointer to `save_t' type");
 
 	s->file.old_fd = -1;
@@ -116,7 +118,7 @@ override_error_output(void *data)
 		goto error;
 	}
 
-	s->file.new_fd = open("/dev/null", O_WRONLY | O_APPEND, mode);
+	s->file.new_fd = open("/dev/null", flags, 0222);
 	if (s->file.new_fd < 0) {
 		local_errno = errno;
 
@@ -128,10 +130,12 @@ override_error_output(void *data)
 		safe_close(s->file.old_fd);
 		goto error;
 	}
+
 	if (safe_cloexec(s->file.new_fd) < 0) {
 		local_errno = errno;
 		goto error;
 	}
+
 	if (dup2(s->file.new_fd, fileno(stderr)) < 0) {
 		local_errno = errno;
 		goto error;
@@ -146,13 +150,13 @@ error:
 	return -1;
 }
 
-int
+static int
 restore_error_output(void *data)
 {
 	int local_errno;
 	save_t *s = data;
 
-	assert(s != NULL && \
+	assert(s != NULL &&
 	       "Must be a valid pointer to `save_t' type");
 
 	if (s->file.old_fd < 0 && s->status != 0)
@@ -208,6 +212,14 @@ magic_errno_wrapper(magic_t magic)
 inline const char*
 magic_getpath_wrapper(void)
 {
+	/*
+	 * The second argument translates to same value as the
+	 * FILE_LOAD constant, which when used results in this
+	 * function calling the get_default_magic() internally.
+	 *
+	 * N.B. magic_getpath() also honors the "MAGIC"
+	 * environment variable."
+	 */
 	return magic_getpath(NULL, 0);
 }
 
@@ -270,15 +282,21 @@ inline int
 magic_load_wrapper(magic_t magic, const char *magic_file, int flags)
 {
 	int rv;
+
 	MAGIC_FUNCTION(magic_load, rv, flags, magic, magic_file);
+
 	return rv;
 }
 
 inline int
-magic_load_buffers_wrapper(magic_t magic, void **buffers, size_t *sizes, size_t count, int flags)
+magic_load_buffers_wrapper(magic_t magic, void **buffers, size_t *sizes,
+			   size_t count, int flags)
 {
 	int rv;
-	MAGIC_FUNCTION(magic_load_buffers, rv, flags, magic, buffers, sizes, count);
+
+	MAGIC_FUNCTION(magic_load_buffers, rv, flags, magic, buffers, sizes,
+		       count);
+
 	return rv;
 }
 
@@ -286,7 +304,9 @@ inline int
 magic_compile_wrapper(magic_t magic, const char *magic_file, int flags)
 {
 	int rv;
+
 	MAGIC_FUNCTION(magic_compile, rv, flags, magic, magic_file);
+
 	return rv;
 }
 
@@ -294,15 +314,19 @@ inline int
 magic_check_wrapper(magic_t magic, const char *magic_file, int flags)
 {
 	int rv;
+
 	MAGIC_FUNCTION(magic_check, rv, flags, magic, magic_file);
+
 	return rv;
 }
 
 inline const char*
-magic_file_wrapper(magic_t magic, const char* filename, int flags)
+magic_file_wrapper(magic_t magic, const char *filename, int flags)
 {
 	const char *cstring;
+
 	MAGIC_FUNCTION(magic_file, cstring, flags, magic, filename);
+
 	return cstring;
 }
 
@@ -310,7 +334,9 @@ inline const char*
 magic_buffer_wrapper(magic_t magic, const void *buffer, size_t size, int flags)
 {
 	const char *cstring;
+
 	MAGIC_FUNCTION(magic_buffer, cstring, flags, magic, buffer, size);
+
 	return cstring;
 }
 

data/ext/magic/functions.h

@@ -9,7 +9,7 @@ extern "C" {
 
 #define MAGIC_FUNCTION(f, r, x, ...)			 \
 	do {						 \
-		if ((x) & MAGIC_DEBUG)			 \
+		if ((x) & (MAGIC_DEBUG | MAGIC_CHECK))   \
 			r = f(__VA_ARGS__);		 \
 		else {					 \
 			save_t __##f;			 \
@@ -17,7 +17,7 @@ extern "C" {
 			r = f(__VA_ARGS__);		 \
 			restore_error_output(&(__##f));  \
 		}					 \
-	} while(0)
+	} while (0)
 
 typedef struct file_data {
 	fpos_t position;
@@ -33,30 +33,36 @@ typedef struct save {
 extern magic_t magic_open_wrapper(int flags);
 extern void magic_close_wrapper(magic_t magic);
 
-extern const char* magic_error_wrapper(magic_t magic);
+extern const char *magic_error_wrapper(magic_t magic);
 extern int magic_errno_wrapper(magic_t magic);
 
-extern const char* magic_getpath_wrapper(void);
+extern const char *magic_getpath_wrapper(void);
 
-extern int magic_getparam_wrapper(magic_t magic, int parameter, void *value);
+extern int magic_getparam_wrapper(magic_t magic, int parameter,
+				  void *value);
 extern int magic_setparam_wrapper(magic_t magic, int parameter,
 				  const void *value);
 
 extern int magic_getflags_wrapper(magic_t magic);
 extern int magic_setflags_wrapper(magic_t magic, int flags);
 
-extern int magic_load_wrapper(magic_t magic, const char *magic_file, int flags);
+extern int magic_load_wrapper(magic_t magic, const char *magic_file,
+			      int flags);
 extern int magic_load_buffers_wrapper(magic_t magic, void **buffers,
-				      size_t *sizes, size_t count, int flags);
+				      size_t *sizes, size_t count,
+				      int flags);
 
-extern int magic_compile_wrapper(magic_t magic, const char *magic_file, int flags);
-extern int magic_check_wrapper(magic_t magic, const char *magic_file, int flags);
+extern int magic_compile_wrapper(magic_t magic, const char *magic_file,
+				 int flags);
+extern int magic_check_wrapper(magic_t magic, const char *magic_file,
+			       int flags);
 
-extern const char* magic_file_wrapper(magic_t magic, const char *filename,
+extern const char *magic_file_wrapper(magic_t magic, const char *filename,
 				      int flags);
-extern const char* magic_buffer_wrapper(magic_t magic, const void *buffer,
+extern const char *magic_buffer_wrapper(magic_t magic, const void *buffer,
 					size_t size, int flags);
-extern const char* magic_descriptor_wrapper(magic_t magic, int fd, int flags);
+extern const char *magic_descriptor_wrapper(magic_t magic, int fd,
+					    int flags);
 
 extern int magic_version_wrapper(void);