extzstd 0.2 → 0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +13 -0
  3. data/README.md +17 -14
  4. data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/Makefile +99 -53
  7. data/contrib/zstd/README.md +59 -39
  8. data/contrib/zstd/TESTING.md +1 -1
  9. data/contrib/zstd/appveyor.yml +17 -6
  10. data/contrib/zstd/lib/BUCK +29 -2
  11. data/contrib/zstd/lib/Makefile +118 -21
  12. data/contrib/zstd/lib/README.md +84 -44
  13. data/contrib/zstd/lib/common/bitstream.h +17 -33
  14. data/contrib/zstd/lib/common/compiler.h +62 -8
  15. data/contrib/zstd/lib/common/cpu.h +215 -0
  16. data/contrib/zstd/lib/common/debug.c +44 -0
  17. data/contrib/zstd/lib/common/debug.h +134 -0
  18. data/contrib/zstd/lib/common/entropy_common.c +16 -1
  19. data/contrib/zstd/lib/common/error_private.c +7 -0
  20. data/contrib/zstd/lib/common/fse.h +48 -44
  21. data/contrib/zstd/lib/common/fse_decompress.c +3 -3
  22. data/contrib/zstd/lib/common/huf.h +169 -113
  23. data/contrib/zstd/lib/common/mem.h +20 -2
  24. data/contrib/zstd/lib/common/pool.c +135 -49
  25. data/contrib/zstd/lib/common/pool.h +40 -21
  26. data/contrib/zstd/lib/common/threading.c +2 -2
  27. data/contrib/zstd/lib/common/threading.h +12 -12
  28. data/contrib/zstd/lib/common/xxhash.c +3 -2
  29. data/contrib/zstd/lib/common/zstd_common.c +3 -6
  30. data/contrib/zstd/lib/common/zstd_errors.h +17 -7
  31. data/contrib/zstd/lib/common/zstd_internal.h +76 -48
  32. data/contrib/zstd/lib/compress/fse_compress.c +89 -209
  33. data/contrib/zstd/lib/compress/hist.c +203 -0
  34. data/contrib/zstd/lib/compress/hist.h +95 -0
  35. data/contrib/zstd/lib/compress/huf_compress.c +188 -80
  36. data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
  37. data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
  38. data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
  39. data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
  40. data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
  41. data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
  46. data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
  47. data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
  48. data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
  49. data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
  50. data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
  51. data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
  52. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  53. data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
  55. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  56. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
  57. data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
  58. data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
  59. data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
  60. data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
  61. data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
  62. data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
  63. data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
  64. data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
  65. data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
  66. data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
  67. data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
  68. data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
  69. data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
  70. data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
  71. data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
  72. data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
  73. data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
  74. data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
  75. data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
  76. data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
  77. data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
  78. data/contrib/zstd/lib/zstd.h +1346 -832
  79. data/ext/extzstd.c +27 -19
  80. data/ext/extzstd_stream.c +20 -4
  81. data/ext/zstd_compress.c +1 -0
  82. data/ext/zstd_decompress.c +4 -0
  83. data/ext/zstd_dictbuilder.c +4 -0
  84. data/ext/zstd_dictbuilder_fastcover.c +5 -0
  85. data/lib/extzstd.rb +52 -220
  86. data/lib/extzstd/version.rb +1 -1
  87. metadata +21 -7
  88. data/contrib/zstd/circle.yml +0 -63
@@ -35,13 +35,18 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
35
35
  size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv04_isError())
43
- */
44
- size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
@@ -491,6 +491,8 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
491
491
 
492
492
  #define WILDCOPY_OVERLENGTH 8
493
493
 
494
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
495
+
494
496
  typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
495
497
 
496
498
 
@@ -836,7 +838,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
836
838
  bitD->bitsConsumed += nbBits;
837
839
  }
838
840
 
839
- MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
841
+ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
840
842
  {
841
843
  size_t value = BITv05_lookBits(bitD, nbBits);
842
844
  BITv05_skipBits(bitD, nbBits);
@@ -845,7 +847,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
845
847
 
846
848
  /*!BITv05_readBitsFast :
847
849
  * unsafe version; only works only if nbBits >= 1 */
848
- MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
850
+ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
849
851
  {
850
852
  size_t value = BITv05_lookBitsFast(bitD, nbBits);
851
853
  BITv05_skipBits(bitD, nbBits);
@@ -1162,7 +1164,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
1162
1164
  /* **************************************************************
1163
1165
  * Complex types
1164
1166
  ****************************************************************/
1165
- typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
1167
+ typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
1166
1168
 
1167
1169
 
1168
1170
  /* **************************************************************
@@ -1224,6 +1226,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
1224
1226
  if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1225
1227
 
1226
1228
  /* Init, lay down lowprob symbols */
1229
+ memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1227
1230
  DTableH.tableLog = (U16)tableLog;
1228
1231
  for (s=0; s<=maxSymbolValue; s++) {
1229
1232
  if (normalizedCounter[s]==-1) {
@@ -2190,7 +2193,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
2190
2193
  }
2191
2194
  }
2192
2195
 
2193
- size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2196
+ size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
2194
2197
  {
2195
2198
  BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
2196
2199
  sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
@@ -2204,7 +2207,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2204
2207
  void* dtPtr = DTable;
2205
2208
  HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
2206
2209
 
2207
- HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2210
+ HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
2208
2211
  if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2209
2212
  //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2210
2213
 
@@ -2331,7 +2334,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
2331
2334
  size_t HUFv05_decompress1X4_usingDTable(
2332
2335
  void* dst, size_t dstSize,
2333
2336
  const void* cSrc, size_t cSrcSize,
2334
- const U32* DTable)
2337
+ const unsigned* DTable)
2335
2338
  {
2336
2339
  const BYTE* const istart = (const BYTE*) cSrc;
2337
2340
  BYTE* const ostart = (BYTE*) dst;
@@ -2374,7 +2377,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
2374
2377
  size_t HUFv05_decompress4X4_usingDTable(
2375
2378
  void* dst, size_t dstSize,
2376
2379
  const void* cSrc, size_t cSrcSize,
2377
- const U32* DTable)
2380
+ const unsigned* DTable)
2378
2381
  {
2379
2382
  if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
2380
2383
 
@@ -2658,6 +2661,7 @@ struct ZSTDv05_DCtx_s
2658
2661
  BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
2659
2662
  }; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
2660
2663
 
2664
+ size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
2661
2665
  size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
2662
2666
 
2663
2667
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
@@ -2822,7 +2826,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
2822
2826
  }
2823
2827
 
2824
2828
 
2825
- size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2829
+ static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2826
2830
  {
2827
2831
  const BYTE* const in = (const BYTE* const)src;
2828
2832
  BYTE headerFlags;
@@ -2845,6 +2849,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
2845
2849
 
2846
2850
  static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2847
2851
  {
2852
+ if (dst==NULL) return ERROR(dstSize_tooSmall);
2848
2853
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2849
2854
  memcpy(dst, src, srcSize);
2850
2855
  return srcSize;
@@ -2853,8 +2858,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
2853
2858
 
2854
2859
  /*! ZSTDv05_decodeLiteralsBlock() :
2855
2860
  @return : nb of bytes read from src (< srcSize ) */
2856
- size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2857
- const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2861
+ static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2862
+ const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2858
2863
  {
2859
2864
  const BYTE* const istart = (const BYTE*) src;
2860
2865
 
@@ -2988,7 +2993,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2988
2993
  }
2989
2994
 
2990
2995
 
2991
- size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
2996
+ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
2992
2997
  FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
2993
2998
  const void* src, size_t srcSize, U32 flagStaticTable)
2994
2999
  {
@@ -2996,7 +3001,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
2996
3001
  const BYTE* ip = istart;
2997
3002
  const BYTE* const iend = istart + srcSize;
2998
3003
  U32 LLtype, Offtype, MLtype;
2999
- U32 LLlog, Offlog, MLlog;
3004
+ unsigned LLlog, Offlog, MLlog;
3000
3005
  size_t dumpsLength;
3001
3006
 
3002
3007
  /* check */
@@ -3054,7 +3059,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3054
3059
  break;
3055
3060
  case FSEv05_ENCODING_DYNAMIC :
3056
3061
  default : /* impossible */
3057
- { U32 max = MaxLL;
3062
+ { unsigned max = MaxLL;
3058
3063
  headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
3059
3064
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3060
3065
  if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
@@ -3078,7 +3083,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3078
3083
  break;
3079
3084
  case FSEv05_ENCODING_DYNAMIC :
3080
3085
  default : /* impossible */
3081
- { U32 max = MaxOff;
3086
+ { unsigned max = MaxOff;
3082
3087
  headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
3083
3088
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3084
3089
  if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
@@ -3102,7 +3107,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3102
3107
  break;
3103
3108
  case FSEv05_ENCODING_DYNAMIC :
3104
3109
  default : /* impossible */
3105
- { U32 max = MaxML;
3110
+ { unsigned max = MaxML;
3106
3111
  headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
3107
3112
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3108
3113
  if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
@@ -3214,7 +3219,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3214
3219
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
3215
3220
  {
3216
3221
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3217
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3222
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3218
3223
  BYTE* const oLitEnd = op + sequence.litLength;
3219
3224
  const size_t sequenceLength = sequence.litLength + sequence.matchLength;
3220
3225
  BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
@@ -3297,14 +3302,14 @@ static size_t ZSTDv05_decompressSequences(
3297
3302
  BYTE* const ostart = (BYTE* const)dst;
3298
3303
  BYTE* op = ostart;
3299
3304
  BYTE* const oend = ostart + maxDstSize;
3300
- size_t errorCode, dumpsLength;
3305
+ size_t errorCode, dumpsLength=0;
3301
3306
  const BYTE* litPtr = dctx->litPtr;
3302
3307
  const BYTE* const litEnd = litPtr + dctx->litSize;
3303
- int nbSeq;
3304
- const BYTE* dumps;
3305
- U32* DTableLL = dctx->LLTable;
3306
- U32* DTableML = dctx->MLTable;
3307
- U32* DTableOffb = dctx->OffTable;
3308
+ int nbSeq=0;
3309
+ const BYTE* dumps = NULL;
3310
+ unsigned* DTableLL = dctx->LLTable;
3311
+ unsigned* DTableML = dctx->MLTable;
3312
+ unsigned* DTableOffb = dctx->OffTable;
3308
3313
  const BYTE* const base = (const BYTE*) (dctx->base);
3309
3314
  const BYTE* const vBase = (const BYTE*) (dctx->vBase);
3310
3315
  const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
@@ -3410,10 +3415,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
3410
3415
  BYTE* const oend = ostart + maxDstSize;
3411
3416
  size_t remainingSize = srcSize;
3412
3417
  blockProperties_t blockProperties;
3418
+ memset(&blockProperties, 0, sizeof(blockProperties));
3413
3419
 
3414
3420
  /* Frame Header */
3415
- {
3416
- size_t frameHeaderSize;
3421
+ { size_t frameHeaderSize;
3417
3422
  if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
3418
3423
  frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
3419
3424
  if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
@@ -3505,34 +3510,57 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3505
3510
  #endif
3506
3511
  }
3507
3512
 
3508
- size_t ZSTDv05_findFrameCompressedSize(const void *src, size_t srcSize)
3513
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3514
+ assumes `cSize` and `dBound` are _not_ NULL */
3515
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3516
+ {
3517
+ *cSize = ret;
3518
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3519
+ }
3520
+
3521
+ void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3509
3522
  {
3510
3523
  const BYTE* ip = (const BYTE*)src;
3511
3524
  size_t remainingSize = srcSize;
3525
+ size_t nbBlocks = 0;
3512
3526
  blockProperties_t blockProperties;
3513
3527
 
3514
3528
  /* Frame Header */
3515
- if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong);
3516
- if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown);
3529
+ if (srcSize < ZSTDv05_frameHeaderSize_min) {
3530
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3531
+ return;
3532
+ }
3533
+ if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) {
3534
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3535
+ return;
3536
+ }
3517
3537
  ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
3518
3538
 
3519
3539
  /* Loop on each block */
3520
3540
  while (1)
3521
3541
  {
3522
3542
  size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
3523
- if (ZSTDv05_isError(cBlockSize)) return cBlockSize;
3543
+ if (ZSTDv05_isError(cBlockSize)) {
3544
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3545
+ return;
3546
+ }
3524
3547
 
3525
3548
  ip += ZSTDv05_blockHeaderSize;
3526
3549
  remainingSize -= ZSTDv05_blockHeaderSize;
3527
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3550
+ if (cBlockSize > remainingSize) {
3551
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3552
+ return;
3553
+ }
3528
3554
 
3529
3555
  if (cBlockSize == 0) break; /* bt_end */
3530
3556
 
3531
3557
  ip += cBlockSize;
3532
3558
  remainingSize -= cBlockSize;
3559
+ nbBlocks++;
3533
3560
  }
3534
3561
 
3535
- return ip - (const BYTE*)src;
3562
+ *cSize = ip - (const BYTE*)src;
3563
+ *dBound = nbBlocks * BLOCKSIZE;
3536
3564
  }
3537
3565
 
3538
3566
  /* ******************************
@@ -3630,7 +3658,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
3630
3658
  {
3631
3659
  size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
3632
3660
  short offcodeNCount[MaxOff+1];
3633
- U32 offcodeMaxValue=MaxOff, offcodeLog;
3661
+ unsigned offcodeMaxValue=MaxOff, offcodeLog;
3634
3662
  short matchlengthNCount[MaxML+1];
3635
3663
  unsigned matchlengthMaxValue = MaxML, matchlengthLog;
3636
3664
  short litlengthNCount[MaxLL+1];
@@ -33,13 +33,18 @@ extern "C" {
33
33
  size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
34
34
  const void* src, size_t compressedSize);
35
35
 
36
- /**
37
- ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
38
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
39
- return : the number of bytes that would be read to decompress this frame
40
- or an errorCode if it fails (which can be tested using ZSTDv05_isError())
41
- */
42
- size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
36
+ /**
37
+ ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
38
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
39
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
40
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
41
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
42
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
43
+
44
+ note : assumes `cSize` and `dBound` are _not_ NULL.
45
+ */
46
+ void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
47
+ size_t* cSize, unsigned long long* dBound);
43
48
 
44
49
  /* *************************************
45
50
  * Helper functions
@@ -189,7 +189,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
189
189
  {
190
190
  #if defined(_MSC_VER) /* Visual Studio */
191
191
  return _byteswap_ulong(in);
192
- #elif defined (__GNUC__)
192
+ #elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
193
193
  return __builtin_bswap32(in);
194
194
  #else
195
195
  return ((in << 24) & 0xff000000 ) |
@@ -203,7 +203,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
203
203
  {
204
204
  #if defined(_MSC_VER) /* Visual Studio */
205
205
  return _byteswap_uint64(in);
206
- #elif defined (__GNUC__)
206
+ #elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
207
207
  return __builtin_bswap64(in);
208
208
  #else
209
209
  return ((in << 56) & 0xff00000000000000ULL) |
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
506
506
  #define FSEv06_ENCODING_STATIC 2
507
507
  #define FSEv06_ENCODING_DYNAMIC 3
508
508
 
509
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
510
+
509
511
  static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
510
512
  1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
511
513
  13,14,15,16 };
@@ -1250,9 +1252,7 @@ const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
1250
1252
  /* **************************************************************
1251
1253
  * HUF Error Management
1252
1254
  ****************************************************************/
1253
- unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
1254
-
1255
- const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
1255
+ static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
1256
1256
 
1257
1257
 
1258
1258
  /*-**************************************************************
@@ -2823,7 +2823,8 @@ struct ZSTDv06_DCtx_s
2823
2823
  BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
2824
2824
  }; /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
2825
2825
 
2826
- size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); } /* non published interface */
2826
+ size_t ZSTDv06_sizeofDCtx (void); /* Hidden declaration */
2827
+ size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }
2827
2828
 
2828
2829
  size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
2829
2830
  {
@@ -3022,7 +3023,7 @@ typedef struct
3022
3023
 
3023
3024
  /*! ZSTDv06_getcBlockSize() :
3024
3025
  * Provides the size of compressed block from block header `src` */
3025
- size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3026
+ static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3026
3027
  {
3027
3028
  const BYTE* const in = (const BYTE* const)src;
3028
3029
  U32 cSize;
@@ -3041,6 +3042,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
3041
3042
 
3042
3043
  static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3043
3044
  {
3045
+ if (dst==NULL) return ERROR(dstSize_tooSmall);
3044
3046
  if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
3045
3047
  memcpy(dst, src, srcSize);
3046
3048
  return srcSize;
@@ -3049,7 +3051,7 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
3049
3051
 
3050
3052
  /*! ZSTDv06_decodeLiteralsBlock() :
3051
3053
  @return : nb of bytes read from src (< srcSize ) */
3052
- size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3054
+ static size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3053
3055
  const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
3054
3056
  {
3055
3057
  const BYTE* const istart = (const BYTE*) src;
@@ -3183,7 +3185,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3183
3185
  @return : nb bytes read from src,
3184
3186
  or an error code if it fails, testable with ZSTDv06_isError()
3185
3187
  */
3186
- size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
3188
+ static size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
3187
3189
  const void* src, size_t srcSize,
3188
3190
  const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
3189
3191
  {
@@ -3213,7 +3215,7 @@ size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLo
3213
3215
  }
3214
3216
 
3215
3217
 
3216
- size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3218
+ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3217
3219
  FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
3218
3220
  const void* src, size_t srcSize)
3219
3221
  {
@@ -3358,7 +3360,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
3358
3360
  }
3359
3361
 
3360
3362
 
3361
- size_t ZSTDv06_execSequence(BYTE* op,
3363
+ static size_t ZSTDv06_execSequence(BYTE* op,
3362
3364
  BYTE* const oend, seq_t sequence,
3363
3365
  const BYTE** litPtr, const BYTE* const litLimit,
3364
3366
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@@ -3406,7 +3408,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
3406
3408
  if (sequence.offset < 8) {
3407
3409
  /* close range match, overlap */
3408
3410
  static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3409
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3411
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3410
3412
  int const sub2 = dec64table[sequence.offset];
3411
3413
  op[0] = match[0];
3412
3414
  op[1] = match[1];
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
3654
3656
  #endif
3655
3657
  }
3656
3658
 
3657
- size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
3659
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3660
+ assumes `cSize` and `dBound` are _not_ NULL */
3661
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3662
+ {
3663
+ *cSize = ret;
3664
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3665
+ }
3666
+
3667
+ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3658
3668
  {
3659
3669
  const BYTE* ip = (const BYTE*)src;
3660
3670
  size_t remainingSize = srcSize;
3671
+ size_t nbBlocks = 0;
3661
3672
  blockProperties_t blockProperties = { bt_compressed, 0 };
3662
3673
 
3663
3674
  /* Frame Header */
3664
3675
  { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
3665
- if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
3666
- if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
3667
- if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
3676
+ if (ZSTDv06_isError(frameHeaderSize)) {
3677
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3678
+ return;
3679
+ }
3680
+ if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
3681
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3682
+ return;
3683
+ }
3684
+ if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
3685
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3686
+ return;
3687
+ }
3668
3688
  ip += frameHeaderSize; remainingSize -= frameHeaderSize;
3669
3689
  }
3670
3690
 
3671
3691
  /* Loop on each block */
3672
3692
  while (1) {
3673
3693
  size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
3674
- if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
3694
+ if (ZSTDv06_isError(cBlockSize)) {
3695
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3696
+ return;
3697
+ }
3675
3698
 
3676
3699
  ip += ZSTDv06_blockHeaderSize;
3677
3700
  remainingSize -= ZSTDv06_blockHeaderSize;
3678
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3701
+ if (cBlockSize > remainingSize) {
3702
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3703
+ return;
3704
+ }
3679
3705
 
3680
3706
  if (cBlockSize == 0) break; /* bt_end */
3681
3707
 
3682
3708
  ip += cBlockSize;
3683
3709
  remainingSize -= cBlockSize;
3710
+ nbBlocks++;
3684
3711
  }
3685
3712
 
3686
- return ip - (const BYTE*)src;
3713
+ *cSize = ip - (const BYTE*)src;
3714
+ *dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
3687
3715
  }
3688
3716
 
3689
3717
  /*_******************************
@@ -4006,7 +4034,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4006
4034
  if (ZSTDv06_isError(hSize)) return hSize;
4007
4035
  if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
4008
4036
  memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
4009
- zbd->lhSize += iend-ip; ip = iend; notDone = 0;
4037
+ zbd->lhSize += iend-ip;
4010
4038
  *dstCapacityPtr = 0;
4011
4039
  return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
4012
4040
  }