extzstd 0.3.1 → 0.3.2

data/contrib/zstd/CONTRIBUTING.md

@@ -126,6 +126,20 @@ just `contrib/largeNbDicts` and nothing else, you can run:
 scan-build make -C contrib/largeNbDicts largeNbDicts
 ```
 
+### Pitfalls of static analysis
+`scan-build` is part of our regular CI suite. Other static analyzers are not.
+
+It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are :
+
+- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low.
+- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload.
+- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to :
+   + torteous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers.
+   + sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time.
+- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer.
+
+This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted.
+
 ## Performance
 Performance is extremely important for zstd and we only merge pull requests whose performance
 landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.

data/contrib/zstd/Makefile

@@ -8,6 +8,9 @@
 # You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
+# verbose mode (print commands) on V=1 or VERBOSE=1
+Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
+
 PRGDIR   = programs
 ZSTDDIR  = lib
 BUILDIR  = build
@@ -28,9 +31,9 @@ VOID = /dev/null
 TARGET_SYSTEM ?= $(OS)
 
 ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
-EXT =.exe
+  EXT =.exe
 else
-EXT =
+  EXT =
 endif
 
 ## default: Build lib-release and zstd-release
@@ -46,8 +49,8 @@ allmost: allzstd zlibwrapper
 # skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
 allzstd: lib-all
-	$(MAKE) -C $(PRGDIR) all
-	$(MAKE) -C $(TESTDIR) all
+	$(Q)$(MAKE) -C $(PRGDIR) all
+	$(Q)$(MAKE) -C $(TESTDIR) all
 
 .PHONY: all32
 all32:
@@ -55,18 +58,19 @@ all32:
 	$(MAKE) -C $(TESTDIR) all32
 
 .PHONY: lib lib-release libzstd.a
+lib-all : lib
 lib lib-release lib-all :
-	@$(MAKE) -C $(ZSTDDIR) $@
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
 zstd zstd-release:
-	@$(MAKE) -C $(PRGDIR) $@
-	cp $(PRGDIR)/zstd$(EXT) .
+	$(Q)$(MAKE) -C $(PRGDIR) $@
+	$(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT)
 
 .PHONY: zstdmt
 zstdmt:
-	@$(MAKE) -C $(PRGDIR) $@
-	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+	$(Q)$(MAKE) -C $(PRGDIR) $@
+	$(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
 
 .PHONY: zlibwrapper
 zlibwrapper: lib
@@ -75,16 +79,16 @@ zlibwrapper: lib
 ## test: run long-duration tests
 .PHONY: test
 DEBUGLEVEL ?= 1
-test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
+test: MOREFLAGS += -g -Werror
 test:
-	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
+	DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@
-	ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder test
+	ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
 
 ## shortest: same as `make check`
 .PHONY: shortest
 shortest:
-	$(MAKE) -C $(TESTDIR) $@
+	$(Q)$(MAKE) -C $(TESTDIR) $@
 
 ## check: run basic tests for `zstd` cli
 .PHONY: check
@@ -97,10 +101,10 @@ automated_benchmarking:
 .PHONY: benchmarking
 benchmarking: automated_benchmarking
 
-## examples: build all examples in `/examples` directory
+## examples: build all examples in `examples/` directory
 .PHONY: examples
 examples: lib
-	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
+	$(MAKE) -C examples all
 
 ## manual: generate API documentation in html format
 .PHONY: manual
@@ -117,6 +121,7 @@ man:
 contrib: lib
 	$(MAKE) -C contrib/pzstd all
 	$(MAKE) -C contrib/seekable_format/examples all
+	$(MAKE) -C contrib/seekable_format/tests test
 	$(MAKE) -C contrib/largeNbDicts all
 	cd contrib/single_file_libs/ ; ./build_decoder_test.sh
 	cd contrib/single_file_libs/ ; ./build_library_test.sh
@@ -127,17 +132,18 @@ cleanTabs:
 
 .PHONY: clean
 clean:
-	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
-	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
-	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
-	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
-	@$(MAKE) -C examples/ $@ > $(VOID)
-	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
-	@$(MAKE) -C contrib/pzstd $@ > $(VOID)
-	@$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
-	@$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
-	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
-	@$(RM) -r lz4
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C examples/ $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
+	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
+	$(Q)$(RM) -r lz4
 	@echo Cleaning completed
 
 #------------------------------------------------------------------------------
@@ -161,7 +167,7 @@ EGREP = egrep $(EGREP_OPTIONS)
 ## list: Print all targets and their descriptions (if provided)
 .PHONY: list
 list:
-	@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
+	$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
 		| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
 		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
 	{ \
@@ -176,13 +182,13 @@ list:
 
 .PHONY: install armtest usan asan uasan
 install:
-	@$(MAKE) -C $(ZSTDDIR) $@
-	@$(MAKE) -C $(PRGDIR) $@
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@
+	$(Q)$(MAKE) -C $(PRGDIR) $@
 
 .PHONY: uninstall
 uninstall:
-	@$(MAKE) -C $(ZSTDDIR) $@
-	@$(MAKE) -C $(PRGDIR) $@
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@
+	$(Q)$(MAKE) -C $(PRGDIR) $@
 
 .PHONY: travis-install
 travis-install:

data/contrib/zstd/README.md

@@ -176,6 +176,12 @@ Going into `build` directory, you will find additional possibilities:
 You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
 The output binary will be in `buck-out/gen/programs/`.
 
+## Testing
+
+You can run quick local smoke tests by executing the `playTest.sh` script from the `src/tests` directory.
+Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the zstd and datagen binary.
+For information on CI testing, please refer to TESTING.md
+
 ## Status
 
 Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.

data/contrib/zstd/appveyor.yml

@@ -162,6 +162,8 @@
   - if [%TEST%]==[cmake] (
       mkdir build\cmake\build &&
       cd build\cmake\build &&
+      SET FUZZERTEST=-T2mn &&
+      SET ZSTREAM_TESTTIME=-T2mn &&
       cmake -G "Visual Studio 14 2015 Win64" .. &&
       cd ..\..\.. &&
       make clean
@@ -194,7 +196,7 @@
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
+      SCRIPT:   "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x86"
@@ -285,5 +287,6 @@
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%HOST%]==[mingw] (
       set "CC=%COMPILER%" &&
+      make clean &&
       make check
     )

data/contrib/zstd/lib/Makefile

@@ -8,15 +8,16 @@
 # You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
-Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
+.PHONY: default
+default: lib-release
 
-# When cross-compiling from linux to windows, you might
-# need to specify this as "Windows." Fedora build fails
-# without it.
-#
-# Note: mingw-w64 build from linux to windows does not
-# fail on other tested distros (ubuntu, debian) even
-# without manually specifying the TARGET_SYSTEM.
+# define silent mode as default (verbose mode with V=1 or VERBOSE=1)
+$(V)$(VERBOSE).SILENT:
+
+# When cross-compiling from linux to windows,
+# one might need to specify TARGET_SYSTEM as "Windows."
+# Building from Fedora fails without it.
+# (but Ubuntu and Debian don't need to set anything)
 TARGET_SYSTEM ?= $(OS)
 
 # Version numbers
@@ -31,24 +32,49 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
 VERSION?= $(LIBVER)
 CCVER := $(shell $(CC) --version)
 
-CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
+# ZSTD_LIB_MINIFY is a helper variable that
+# configures a bunch of other variables to space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
+  ZSTD_LEGACY_SUPPORT ?= 0
+  ZSTD_LIB_DEPRECATED ?= 0
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
+  ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
+  ZSTD_NO_INLINE ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
+ifneq ($(HAVE_CC_OZ), 0)
+    # Some compilers (clang) support an even more space-optimized setting.
+    CFLAGS += -Oz
+else
+    CFLAGS += -Os
+endif
+  CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
+            -DDYNAMIC_BMI2=0 -DNDEBUG
+else
+  CFLAGS += -O3
+endif
+
+DEBUGLEVEL ?= 0
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
 ifeq ($(TARGET_SYSTEM),Windows_NT)   # MinGW assumed
-CPPFLAGS   += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
+  CPPFLAGS += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
 endif
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
             -Wredundant-decls -Wmissing-prototypes -Wc++-compat
-CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CFLAGS)
+CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS)
 
 HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 GREP_OPTIONS ?=
 ifeq ($HAVE_COLORNEVER, 1)
-GREP_OPTIONS += --color=never
+  GREP_OPTIONS += --color=never
 endif
 GREP = grep $(GREP_OPTIONS)
+SED_ERE_OPT ?= -E
 
 ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
 ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
@@ -58,30 +84,7 @@ ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
 ZSTD_FILES := $(ZSTDCOMMON_FILES)
 
 ifeq ($(findstring GCC,$(CCVER)),GCC)
-decompress/zstd_decompress_block.o :	CFLAGS+=-fno-tree-vectorize
-endif
-
-# This is a helper variable that configures a bunch of other variables to new,
-# space-optimized defaults.
-ZSTD_LIB_MINIFY ?= 0
-ifneq ($(ZSTD_LIB_MINIFY), 0)
-	HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
-	ZSTD_LEGACY_SUPPORT ?= 0
-	ZSTD_LIB_DEPRECATED ?= 0
-	HUF_FORCE_DECOMPRESS_X1 ?= 1
-	ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
-	ZSTD_NO_INLINE ?= 1
-	ZSTD_STRIP_ERROR_STRINGS ?= 1
-	ifneq ($(HAVE_CC_OZ), 0)
-		# Some compilers (clang) support an even more space-optimized setting.
-		CFLAGS += -Oz
-	else
-		CFLAGS += -Os
-	endif
-	CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
-	          -DDYNAMIC_BMI2=0 -DNDEBUG
-else
-	CFLAGS += -O3
+decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
 endif
 
 # Modules
@@ -103,116 +106,187 @@ ZSTD_NO_INLINE ?= 0
 ZSTD_STRIP_ERROR_STRINGS ?= 0
 
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
-	ZSTD_LIB_DICTBUILDER = 0
-	ZSTD_LIB_DEPRECATED = 0
+  ZSTD_LIB_DICTBUILDER = 0
+  ZSTD_LIB_DEPRECATED = 0
 endif
 
 ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
-	ZSTD_LEGACY_SUPPORT = 0
-	ZSTD_LIB_DEPRECATED = 0
+  ZSTD_LEGACY_SUPPORT = 0
+  ZSTD_LIB_DEPRECATED = 0
 endif
 
 ifneq ($(ZSTD_LIB_COMPRESSION), 0)
-	ZSTD_FILES += $(ZSTDCOMP_FILES)
+  ZSTD_FILES += $(ZSTDCOMP_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
-	ZSTD_FILES += $(ZSTDDECOMP_FILES)
+  ZSTD_FILES += $(ZSTDDECOMP_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DEPRECATED), 0)
-	ZSTD_FILES += $(ZDEPR_FILES)
+  ZSTD_FILES += $(ZDEPR_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
-	ZSTD_FILES += $(ZDICT_FILES)
+  ZSTD_FILES += $(ZDICT_FILES)
 endif
 
 ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
-	CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
 endif
 
 ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
-	CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
 endif
 
 ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
-	CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
 endif
 
 ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
-	CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
 endif
 
 ifneq ($(ZSTD_NO_INLINE), 0)
-	CFLAGS += -DZSTD_NO_INLINE
+  CFLAGS += -DZSTD_NO_INLINE
 endif
 
 ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
-	CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
+  CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
 endif
 
 ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
-	CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
+  CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
 endif
 
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
-	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+  ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
 endif
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 
-ZSTD_OBJ   := $(patsubst %.c,%.o,$(ZSTD_FILES))
+ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
+ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o)
+
+ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated
+vpath %.c $(ZSTD_SUBDIR)
+
+UNAME := $(shell uname)
+
+ifndef BUILD_DIR
+ifeq ($(UNAME), Darwin)
+  HASH ?= md5
+else ifeq ($(UNAME), FreeBSD)
+  HASH ?= gmd5sum
+else ifeq ($(UNAME), OpenBSD)
+  HASH ?= md5
+endif
+HASH ?= md5sum
+
+HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " )
+HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
+ifeq ($(HAVE_HASH),0)
+  $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
+  BUILD_DIR := obj/generic_noconf
+endif
+endif # BUILD_DIR
+
 
 # macOS linker doesn't support -soname, and use different extension
 # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
-ifeq ($(shell uname), Darwin)
-	SHARED_EXT = dylib
-	SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
-	SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
-	SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+ifeq ($(UNAME), Darwin)
+  SHARED_EXT = dylib
+  SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
+  SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
+  SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
-	SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
-	SHARED_EXT = so
-	SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
-	SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
+  SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
+  SHARED_EXT = so
+  SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
+  SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
+SET_CACHE_DIRECTORY = \
+	$(MAKE) --no-print-directory $@ \
+    BUILD_DIR=obj/$(HASH_DIR) \
+    CPPFLAGS="$(CPPFLAGS)" \
+    CFLAGS="$(CFLAGS)" \
+    LDFLAGS="$(LDFLAGS)"
 
-.PHONY: default lib-all all clean install uninstall
 
-default: lib-release
+.PHONY: lib-all all clean install uninstall
 
 # alias
 lib-all: all
 
 all: lib
 
-libzstd.a: ARFLAGS = rcs
-libzstd.a: $(ZSTD_OBJ)
+.PHONY: libzstd.a  # must be run every time
+
+ifndef BUILD_DIR
+# determine BUILD_DIR from compilation flags
+
+libzstd.a:
+	$(SET_CACHE_DIRECTORY)
+
+else
+# BUILD_DIR is defined
+
+ZSTD_STATLIB_DIR := $(BUILD_DIR)/static
+ZSTD_STATLIB := $(ZSTD_STATLIB_DIR)/libzstd.a
+ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
+$(ZSTD_STATLIB): ARFLAGS = rcs
+$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR)
+$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ)
 	@echo compiling static library
-	$(Q)$(AR) $(ARFLAGS) $@ $^
+	$(AR) $(ARFLAGS) $@ $^
+
+libzstd.a: $(ZSTD_STATLIB)
+	cp -f $< $@
+
+endif
 
 ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
 
-LIBZSTD = dll\libzstd.dll
+LIBZSTD = dll/libzstd.dll
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@
+	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@
 
-else
+else  # not Windows
 
 LIBZSTD = libzstd.$(SHARED_EXT_VER)
-$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
-$(LIBZSTD): $(ZSTD_FILES)
+.PHONY: $(LIBZSTD)  # must be run every time
+$(LIBZSTD): CFLAGS += -fPIC
+$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden
+
+ifndef BUILD_DIR
+# determine BUILD_DIR from compilation flags
+
+$(LIBZSTD):
+	$(SET_CACHE_DIRECTORY)
+
+else
+# BUILD_DIR is defined
+
+ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynamic
+ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD)
+ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
+
+$(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR)
+$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
 	@echo compiling dynamic library $(LIBVER)
-	$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	$(Q)ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
-	$(Q)ln -sf $@ libzstd.$(SHARED_EXT)
+	ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
+	ln -sf $@ libzstd.$(SHARED_EXT)
 
-endif
+$(LIBZSTD): $(ZSTD_DYNLIB)
+	cp -f $< $@
+
+endif  # ifndef BUILD_DIR
+endif  # if windows
 
 .PHONY: libzstd
 libzstd : $(LIBZSTD)
@@ -220,18 +294,42 @@ libzstd : $(LIBZSTD)
 .PHONY: lib
 lib : libzstd.a libzstd
 
-.PHONY: lib-mt
+
+# note : do not define lib-mt or lib-release as .PHONY
+# make does not consider implicit pattern rule for .PHONY target
+
 %-mt : CPPFLAGS += -DZSTD_MULTITHREAD
 %-mt : LDFLAGS  += -pthread
 %-mt : %
 	@echo multi-threading build completed
 
-.PHONY: lib-release
 %-release : DEBUGFLAGS :=
 %-release : %
 	@echo release build completed
 
 
+# Generate .h dependencies automatically
+
+DEPFLAGS = -MT $@ -MMD -MP -MF
+
+$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR)
+	@echo CC $@
+	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
+	@echo CC $@
+	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+MKDIR ?= mkdir
+$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
+	$(MKDIR) -p $@
+
+DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d)
+$(DEPFILES):
+
+include $(wildcard $(DEPFILES))
+
+
 # Special case : building library in single-thread mode _and_ without zstdmt_compress.c
 ZSTDMT_FILES = compress/zstdmt_compress.c
 ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
@@ -239,22 +337,24 @@ libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden
 libzstd-nomt: $(ZSTD_NOMT_FILES)
 	@echo compiling single-thread dynamic library $(LIBVER)
 	@echo files : $(ZSTD_NOMT_FILES)
-	$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 
 clean:
-	$(Q)$(RM) -r *.dSYM   # macOS-specific
-	$(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
-	$(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
-	$(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
+	$(RM) -r *.dSYM   # macOS-specific
+	$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+	$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
+	$(RM) -r obj/*
 	@echo Cleaning library completed
 
 #-----------------------------------------------------------------------------
 # make install is validated only for below listed environments
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
+ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
 
 all: libzstd.pc
 
+HAS_EXPLICIT_EXEC_PREFIX := $(if $(or $(EXEC_PREFIX),$(exec_prefix)),1,)
+
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
@@ -268,35 +368,28 @@ LIBDIR      ?= $(libdir)
 includedir  ?= $(PREFIX)/include
 INCLUDEDIR  ?= $(includedir)
 
-PCLIBDIR ?= $(shell echo "$(LIBDIR)" | sed -n -E -e "s@^$(EXEC_PREFIX)(/|$$)@@p")
-PCINCDIR ?= $(shell echo "$(INCLUDEDIR)" | sed -n -E -e "s@^$(PREFIX)(/|$$)@@p")
+PCINCDIR := $(patsubst $(PREFIX)%,%,$(INCLUDEDIR))
+PCLIBDIR := $(patsubst $(EXEC_PREFIX)%,%,$(LIBDIR))
 
-ifeq (,$(PCLIBDIR))
-# Additional prefix check is required, since the empty string is technically a
-# valid PCLIBDIR
-ifeq (,$(shell echo "$(LIBDIR)" | sed -n -E -e "\\@^$(EXEC_PREFIX)(/|$$)@ p"))
-$(error configured libdir ($(LIBDIR)) is outside of prefix ($(PREFIX)), can't generate pkg-config file)
-endif
-endif
+# If we successfully stripped off a prefix, we'll add a reference to the
+# relevant pc variable.
+PCINCPREFIX := $(if $(findstring $(INCLUDEDIR),$(PCINCDIR)),,$${prefix})
+PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
 
-ifeq (,$(PCINCDIR))
-# Additional prefix check is required, since the empty string is technically a
-# valid PCINCDIR
-ifeq (,$(shell echo "$(INCLUDEDIR)" | sed -n -E -e "\\@^$(PREFIX)(/|$$)@ p"))
-$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(EXEC_PREFIX)), can't generate pkg-config file)
-endif
-endif
+# If no explicit EXEC_PREFIX was set by the caller, write it out as a reference
+# to PREFIX, rather than as a resolved value.
+PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})
 
-ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
-PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
+  PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
-PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+  PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif
 
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
+ifneq (,$(filter $(UNAME),SunOS))
+  INSTALL ?= ginstall
 else
-INSTALL ?= install
+  INSTALL ?= install
 endif
 
 INSTALL_PROGRAM ?= $(INSTALL)
@@ -306,9 +399,11 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 libzstd.pc:
 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
-	$(Q)@sed -E -e 's|@PREFIX@|$(PREFIX)|' \
-          -e 's|@LIBDIR@|$(PCLIBDIR)|' \
-          -e 's|@INCLUDEDIR@|$(PCINCDIR)|' \
+	@sed $(SED_ERE_OPT) \
+	        -e 's|@PREFIX@|$(PREFIX)|' \
+	        -e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \
+          -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
+          -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
           -e 's|@VERSION@|$(VERSION)|' \
           $< >$@
 
@@ -316,39 +411,41 @@ install: install-pc install-static install-shared install-includes
 	@echo zstd static and shared library installed
 
 install-pc: libzstd.pc
-	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
-	$(Q)$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	[ -e $(DESTDIR)$(PKGCONFIGDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
+	$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
 
-install-static: libzstd.a
+install-static:
+	# only generate libzstd.a if it's not already present
+	[ -e libzstd.a ] || $(MAKE) libzstd.a-release
+	[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
 	@echo Installing static library
-	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
-	$(Q)$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
+	$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
 
-install-shared: libzstd
+install-shared:
+	# only generate libzstd.so if it's not already present
+	[ -e $(LIBZSTD) ] || $(MAKE) libzstd-release
+	[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
 	@echo Installing shared library
-	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
-	$(Q)$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
-	$(Q)ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	$(Q)ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
+	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 
 install-includes:
+	[ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
 	@echo Installing includes
-	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
-	$(Q)$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
-	$(Q)$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
-	$(Q)$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)     # prototypes generate deprecation warnings
-	$(Q)$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
 
 uninstall:
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
-	$(Q)$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h   # Deprecated streaming functions
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
+	$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
+	$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
+	$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+	$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
 	@echo zstd libraries successfully uninstalled
 
 endif