extlzham 0.0.1.PROTOTYPE3-x86-mingw32

data/contrib/lzham/lzhamcomp/lzham_match_accel.h

@@ -0,0 +1,146 @@
+// File: lzham_match_accel.h
+// See Copyright Notice and license at the end of include/lzham.h
+#pragma once
+#include "lzham_lzbase.h"
+#include "lzham_threading.h"
+
+namespace lzham
+{
+   const uint cMatchAccelMaxSupportedProbes = 128;
+      
+   struct node
+   {
+      uint m_left;
+      uint m_right;
+   };
+   
+   LZHAM_DEFINE_BITWISE_MOVABLE(node);
+   
+#pragma pack(push, 1)      
+   struct dict_match
+   {
+      uint m_dist;
+      uint16 m_len;
+
+      inline uint get_dist() const { return m_dist & 0x7FFFFFFF; }
+      inline uint get_len() const { return m_len + 2; }
+      inline bool is_last() const { return (int)m_dist < 0; }
+   };
+#pragma pack(pop)  
+
+   LZHAM_DEFINE_BITWISE_MOVABLE(dict_match);
+   
+   class search_accelerator
+   {
+   public:
+      search_accelerator();
+
+      // If all_matches is true, the match finder returns all found matches with no filtering.
+      // Otherwise, the finder will tend to return lists of matches with mostly unique lengths.
+      // For each length, it will discard matches with worse distances (in the coding sense).
+      bool init(CLZBase* pLZBase, task_pool* pPool, uint max_helper_threads, uint max_dict_size, uint max_matches, bool all_matches, uint max_probes);
+      
+      void reset();
+      void flush();
+      
+      inline uint get_max_dict_size() const { return m_max_dict_size; }
+      inline uint get_max_dict_size_mask() const { return m_max_dict_size_mask; }
+      inline uint get_cur_dict_size() const { return m_cur_dict_size; }
+      
+      inline uint get_lookahead_pos() const { return m_lookahead_pos; }
+      inline uint get_lookahead_size() const { return m_lookahead_size; }
+      
+      inline uint get_char(int delta_pos) const { return m_dict[(m_lookahead_pos + delta_pos) & m_max_dict_size_mask]; }
+      inline uint get_char(uint cur_dict_pos, int delta_pos) const { return m_dict[(cur_dict_pos + delta_pos) & m_max_dict_size_mask]; }
+      inline const uint8* get_ptr(uint pos) const { return &m_dict[pos]; }
+      
+      uint get_max_helper_threads() const { return m_max_helper_threads; }
+      
+      inline uint operator[](uint pos) const { return m_dict[pos]; }
+            
+      uint get_max_add_bytes() const;
+      bool add_bytes_begin(uint num_bytes, const uint8* pBytes);
+      inline atomic32_t get_num_completed_helper_threads() const { return m_num_completed_helper_threads; }
+      void add_bytes_end();
+
+      // Returns the lookahead's raw position/size/dict_size at the time add_bytes_begin() is called.
+      inline uint get_fill_lookahead_pos() const { return m_fill_lookahead_pos; }
+      inline uint get_fill_lookahead_size() const { return m_fill_lookahead_size; }
+      inline uint get_fill_dict_size() const { return m_fill_dict_size; }
+      
+      uint get_len2_match(uint lookahead_ofs);
+      dict_match* find_matches(uint lookahead_ofs, bool spin = true);
+            
+      void advance_bytes(uint num_bytes);
+      
+      LZHAM_FORCE_INLINE uint get_match_len(uint lookahead_ofs, int dist, uint max_match_len, uint start_match_len = 0) const
+      {
+         LZHAM_ASSERT(lookahead_ofs < m_lookahead_size);
+         LZHAM_ASSERT(start_match_len <= max_match_len);
+         LZHAM_ASSERT(max_match_len <= (get_lookahead_size() - lookahead_ofs));
+
+         const int find_dict_size = m_cur_dict_size + lookahead_ofs;
+         if (dist > find_dict_size)
+            return 0;
+
+         const uint comp_pos = static_cast<uint>((m_lookahead_pos + lookahead_ofs - dist) & m_max_dict_size_mask);
+         const uint lookahead_pos = (m_lookahead_pos + lookahead_ofs) & m_max_dict_size_mask;
+         
+         const uint8* pComp = &m_dict[comp_pos];
+         const uint8* pLookahead = &m_dict[lookahead_pos];
+         
+         uint match_len;
+         for (match_len = start_match_len; match_len < max_match_len; match_len++)
+            if (pComp[match_len] != pLookahead[match_len])
+               break;
+
+         return match_len;
+      }
+                  
+   public:
+      CLZBase* m_pLZBase;
+      task_pool* m_pTask_pool;
+      uint m_max_helper_threads;
+   
+      uint m_max_dict_size;
+      uint m_max_dict_size_mask;
+      
+      uint m_lookahead_pos;
+      uint m_lookahead_size;
+                  
+      uint m_cur_dict_size;
+            
+      lzham::vector<uint8> m_dict;
+      
+      enum { cHashSize = 65536 };
+      lzham::vector<uint> m_hash;
+      lzham::vector<node> m_nodes;
+
+      lzham::vector<dict_match> m_matches;
+      lzham::vector<atomic32_t> m_match_refs;
+      
+      lzham::vector<uint8> m_hash_thread_index;
+      
+      enum { cDigramHashSize = 4096 };
+      lzham::vector<uint> m_digram_hash;
+      lzham::vector<uint> m_digram_next;
+                                          
+      uint m_fill_lookahead_pos;
+      uint m_fill_lookahead_size;
+      uint m_fill_dict_size;
+      
+      uint m_max_probes;
+      uint m_max_matches;
+      
+      bool m_all_matches;
+                  
+      volatile atomic32_t m_next_match_ref;
+      
+      volatile atomic32_t m_num_completed_helper_threads;
+                  
+      void find_all_matches_callback(uint64 data, void* pData_ptr);
+      bool find_all_matches(uint num_bytes);
+      bool find_len2_matches();
+   };
+
+} // namespace lzham

data/contrib/lzham/lzhamcomp/lzham_null_threading.h

@@ -0,0 +1,97 @@
+// File: lzham_task_pool_null.h
+// See Copyright Notice and license at the end of include/lzham.h
+#pragma once
+
+namespace lzham
+{
+   class semaphore
+   {
+      LZHAM_NO_COPY_OR_ASSIGNMENT_OP(semaphore);
+
+   public:
+      inline semaphore(long initialCount = 0, long maximumCount = 1, const char* pName = NULL)
+      {
+         (void)initialCount, (void)maximumCount, (void)pName;
+      }
+
+      inline ~semaphore()
+      {
+      }
+
+      inline void release(long releaseCount = 1, long *pPreviousCount = NULL)
+      {
+         (void)releaseCount, (void)pPreviousCount;
+      }
+
+      inline bool wait(uint32 milliseconds = UINT32_MAX)
+      {
+         (void)milliseconds;
+         return true;
+      }      
+   };   
+
+   class task_pool
+   {
+   public:
+      inline task_pool() { }
+      inline task_pool(uint num_threads) { (void)num_threads; }
+      inline ~task_pool() { }
+
+      inline bool init(uint num_threads) { (void)num_threads; return true; }
+      inline void deinit();
+
+      inline uint get_num_threads() const { return 0; }
+      inline uint get_num_outstanding_tasks() const { return 0; }
+
+      // C-style task callback
+      typedef void (*task_callback_func)(uint64 data, void* pData_ptr);
+      inline bool queue_task(task_callback_func pFunc, uint64 data = 0, void* pData_ptr = NULL)
+      {
+         pFunc(data, pData_ptr);
+         return true;
+      }
+
+      class executable_task
+      {
+      public:
+         virtual void execute_task(uint64 data, void* pData_ptr) = 0;
+      };
+
+      // It's the caller's responsibility to delete pObj within the execute_task() method, if needed!
+      inline bool queue_task(executable_task* pObj, uint64 data = 0, void* pData_ptr = NULL)
+      {
+         pObj->execute_task(data, pData_ptr);
+         return true;
+      }
+
+      template<typename S, typename T>
+      inline bool queue_object_task(S* pObject, T pObject_method, uint64 data = 0, void* pData_ptr = NULL)
+      {
+         (pObject->*pObject_method)(data, pData_ptr);
+         return true;
+      }
+
+      template<typename S, typename T>
+      inline bool queue_multiple_object_tasks(S* pObject, T pObject_method, uint64 first_data, uint num_tasks, void* pData_ptr = NULL)
+      {
+         for (uint i = 0; i < num_tasks; i++)
+         {
+            (pObject->*pObject_method)(first_data + i, pData_ptr);
+         }
+         return true;
+      }
+
+      void join() { }
+   };
+   
+   inline void lzham_sleep(unsigned int milliseconds)
+   {
+      (void)milliseconds;
+   }
+
+   inline uint lzham_get_max_helper_threads()
+   {
+      return 0;
+   }
+
+} // namespace lzham

data/contrib/lzham/lzhamcomp/lzham_pthreads_threading.cpp

@@ -0,0 +1,229 @@
+// File: lzham_task_pool_pthreads.cpp
+//
+// Copyright (c) 2009-2010 Richard Geldreich, Jr. <richgel99@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+#include "lzham_core.h"
+#include "lzham_pthreads_threading.h"
+#include "lzham_timer.h"
+
+#ifdef WIN32
+#include <process.h>
+#endif
+
+#if defined(__GNUC__) && !defined(__APPLE__) && !defined(__MINGW32__) && !defined(__FreeBSD__)
+#include <sys/sysinfo.h>
+#endif
+
+#if LZHAM_USE_PTHREADS_API
+
+#ifdef WIN32
+#pragma comment(lib, "../ext/libpthread/lib/pthreadVC2.lib")
+#endif
+
+namespace lzham
+{
+   task_pool::task_pool() :
+      m_num_threads(0),
+      m_tasks_available(0, 32767),
+      m_num_outstanding_tasks(0),
+      m_exit_flag(false)
+   {
+      utils::zero_object(m_threads);
+   }
+
+   task_pool::task_pool(uint num_threads) :
+      m_num_threads(0),
+      m_tasks_available(0, 32767),
+      m_num_outstanding_tasks(0),
+      m_exit_flag(false)
+   {
+      utils::zero_object(m_threads);
+
+      bool status = init(num_threads);
+      LZHAM_VERIFY(status);
+   }
+
+   task_pool::~task_pool()
+   {
+      deinit();
+   }
+
+   bool task_pool::init(uint num_threads)
+   {
+      LZHAM_ASSERT(num_threads <= cMaxThreads);
+      num_threads = math::minimum<uint>(num_threads, cMaxThreads);
+
+      deinit();
+
+      bool succeeded = true;
+
+      m_num_threads = 0;
+      while (m_num_threads < num_threads)
+      {
+         int status = pthread_create(&m_threads[m_num_threads], NULL, thread_func, this);
+         if (status)
+         {
+            succeeded = false;
+            break;
+         }
+
+         m_num_threads++;
+      }
+
+      if (!succeeded)
+      {
+         deinit();
+         return false;
+      }
+
+      return true;
+   }
+
+   void task_pool::deinit()
+   {
+      if (m_num_threads)
+      {
+         join();
+
+         atomic_exchange32(&m_exit_flag, true);
+
+         m_tasks_available.release(m_num_threads);
+
+         for (uint i = 0; i < m_num_threads; i++)
+            pthread_join(m_threads[i], NULL);
+
+         m_num_threads = 0;
+
+         atomic_exchange32(&m_exit_flag, false);
+      }
+
+      m_task_stack.clear();
+      m_num_outstanding_tasks = 0;
+   }
+
+   bool task_pool::queue_task(task_callback_func pFunc, uint64 data, void* pData_ptr)
+   {
+      LZHAM_ASSERT(m_num_threads);
+      LZHAM_ASSERT(pFunc);
+
+      task tsk;
+      tsk.m_callback = pFunc;
+      tsk.m_data = data;
+      tsk.m_pData_ptr = pData_ptr;
+      tsk.m_flags = 0;
+
+      if (!m_task_stack.try_push(tsk))
+         return false;
+
+      atomic_increment32(&m_num_outstanding_tasks);
+
+      m_tasks_available.release(1);
+
+      return true;
+   }
+
+   // It's the object's responsibility to delete pObj within the execute_task() method, if needed!
+   bool task_pool::queue_task(executable_task* pObj, uint64 data, void* pData_ptr)
+   {
+      LZHAM_ASSERT(m_num_threads);
+      LZHAM_ASSERT(pObj);
+
+      task tsk;
+      tsk.m_pObj = pObj;
+      tsk.m_data = data;
+      tsk.m_pData_ptr = pData_ptr;
+      tsk.m_flags = cTaskFlagObject;
+
+      if (!m_task_stack.try_push(tsk))
+         return false;
+
+      atomic_increment32(&m_num_outstanding_tasks);
+
+      m_tasks_available.release(1);
+
+      return true;
+   }
+
+   void task_pool::process_task(task& tsk)
+   {
+      if (tsk.m_flags & cTaskFlagObject)
+         tsk.m_pObj->execute_task(tsk.m_data, tsk.m_pData_ptr);
+      else
+         tsk.m_callback(tsk.m_data, tsk.m_pData_ptr);
+
+      atomic_decrement32(&m_num_outstanding_tasks);
+   }
+
+   void task_pool::join()
+   {
+      task tsk;
+      while (atomic_add32(&m_num_outstanding_tasks, 0) > 0)
+      {
+         if (m_task_stack.pop(tsk))
+         {
+            process_task(tsk);
+         }
+         else
+         {
+            lzham_sleep(1);
+         }
+      }
+   }
+
+   void * task_pool::thread_func(void *pContext)
+   {
+      task_pool* pPool = static_cast<task_pool*>(pContext);
+      task tsk;
+
+      for ( ; ; )
+      {
+         if (!pPool->m_tasks_available.wait())
+            break;
+
+         if (pPool->m_exit_flag)
+            break;
+
+         if (pPool->m_task_stack.pop(tsk))
+         {
+            pPool->process_task(tsk);
+         }
+      }
+
+      return NULL;
+   }
+
+   uint lzham_get_max_helper_threads()
+   {
+#if defined(__APPLE__) || defined(__FreeBSD__)
+      int num_procs = static_cast<int>(sysconf(_SC_NPROCESSORS_ONLN));
+      return (num_procs >= 1) ? (num_procs - 1) : 0;
+#elif (1)
+      uint num_procs = get_nprocs();
+      return num_procs ? (num_procs - 1) : 0;
+#else
+      printf("TODO: lzham_get_max_helper_threads(): Implement system specific func to determine the max # of helper threads\n");
+      // Just assume a dual-core machine.
+      return 1;
+#endif
+   }
+
+} // namespace lzham
+
+#endif // LZHAM_USE_PTHREADS_API