extzstd 0.2 → 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +13 -0
  3. data/README.md +17 -14
  4. data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/Makefile +99 -53
  7. data/contrib/zstd/README.md +59 -39
  8. data/contrib/zstd/TESTING.md +1 -1
  9. data/contrib/zstd/appveyor.yml +17 -6
  10. data/contrib/zstd/lib/BUCK +29 -2
  11. data/contrib/zstd/lib/Makefile +118 -21
  12. data/contrib/zstd/lib/README.md +84 -44
  13. data/contrib/zstd/lib/common/bitstream.h +17 -33
  14. data/contrib/zstd/lib/common/compiler.h +62 -8
  15. data/contrib/zstd/lib/common/cpu.h +215 -0
  16. data/contrib/zstd/lib/common/debug.c +44 -0
  17. data/contrib/zstd/lib/common/debug.h +134 -0
  18. data/contrib/zstd/lib/common/entropy_common.c +16 -1
  19. data/contrib/zstd/lib/common/error_private.c +7 -0
  20. data/contrib/zstd/lib/common/fse.h +48 -44
  21. data/contrib/zstd/lib/common/fse_decompress.c +3 -3
  22. data/contrib/zstd/lib/common/huf.h +169 -113
  23. data/contrib/zstd/lib/common/mem.h +20 -2
  24. data/contrib/zstd/lib/common/pool.c +135 -49
  25. data/contrib/zstd/lib/common/pool.h +40 -21
  26. data/contrib/zstd/lib/common/threading.c +2 -2
  27. data/contrib/zstd/lib/common/threading.h +12 -12
  28. data/contrib/zstd/lib/common/xxhash.c +3 -2
  29. data/contrib/zstd/lib/common/zstd_common.c +3 -6
  30. data/contrib/zstd/lib/common/zstd_errors.h +17 -7
  31. data/contrib/zstd/lib/common/zstd_internal.h +76 -48
  32. data/contrib/zstd/lib/compress/fse_compress.c +89 -209
  33. data/contrib/zstd/lib/compress/hist.c +203 -0
  34. data/contrib/zstd/lib/compress/hist.h +95 -0
  35. data/contrib/zstd/lib/compress/huf_compress.c +188 -80
  36. data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
  37. data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
  38. data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
  39. data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
  40. data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
  41. data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
  46. data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
  47. data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
  48. data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
  49. data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
  50. data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
  51. data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
  52. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  53. data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
  55. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  56. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
  57. data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
  58. data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
  59. data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
  60. data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
  61. data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
  62. data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
  63. data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
  64. data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
  65. data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
  66. data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
  67. data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
  68. data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
  69. data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
  70. data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
  71. data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
  72. data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
  73. data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
  74. data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
  75. data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
  76. data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
  77. data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
  78. data/contrib/zstd/lib/zstd.h +1346 -832
  79. data/ext/extzstd.c +27 -19
  80. data/ext/extzstd_stream.c +20 -4
  81. data/ext/zstd_compress.c +1 -0
  82. data/ext/zstd_decompress.c +4 -0
  83. data/ext/zstd_dictbuilder.c +4 -0
  84. data/ext/zstd_dictbuilder_fastcover.c +5 -0
  85. data/lib/extzstd.rb +52 -220
  86. data/lib/extzstd/version.rb +1 -1
  87. metadata +21 -7
  88. data/contrib/zstd/circle.yml +0 -63
@@ -35,13 +35,18 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
35
35
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv01_isError())
43
- */
44
- size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
@@ -399,11 +399,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
399
399
  switch(srcSize)
400
400
  {
401
401
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
402
+ /* fallthrough */
402
403
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
404
+ /* fallthrough */
403
405
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
406
+ /* fallthrough */
404
407
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
408
+ /* fallthrough */
405
409
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
410
+ /* fallthrough */
406
411
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
412
+ /* fallthrough */
407
413
  default:;
408
414
  }
409
415
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -2722,6 +2728,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2722
2728
  #define LITERAL_NOENTROPY 63
2723
2729
  #define COMMAND_NOENTROPY 7 /* to remove */
2724
2730
 
2731
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2732
+
2725
2733
  static const size_t ZSTD_blockHeaderSize = 3;
2726
2734
  static const size_t ZSTD_frameHeaderSize = 4;
2727
2735
 
@@ -3090,7 +3098,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3090
3098
  BYTE* const base, BYTE* const oend)
3091
3099
  {
3092
3100
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
3093
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
3101
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
3094
3102
  const BYTE* const ostart = op;
3095
3103
  BYTE* const oLitEnd = op + sequence.litLength;
3096
3104
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -3306,37 +3314,59 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3306
3314
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3307
3315
  }
3308
3316
 
3309
- static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
3317
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3318
+ assumes `cSize` and `dBound` are _not_ NULL */
3319
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3310
3320
  {
3321
+ *cSize = ret;
3322
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3323
+ }
3311
3324
 
3325
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3326
+ {
3312
3327
  const BYTE* ip = (const BYTE*)src;
3313
3328
  size_t remainingSize = srcSize;
3329
+ size_t nbBlocks = 0;
3314
3330
  U32 magicNumber;
3315
3331
  blockProperties_t blockProperties;
3316
3332
 
3317
3333
  /* Frame Header */
3318
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
3334
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
3335
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3336
+ return;
3337
+ }
3319
3338
  magicNumber = MEM_readLE32(src);
3320
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
3339
+ if (magicNumber != ZSTD_magicNumber) {
3340
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3341
+ return;
3342
+ }
3321
3343
  ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3322
3344
 
3323
3345
  /* Loop on each block */
3324
3346
  while (1)
3325
3347
  {
3326
3348
  size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3327
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
3349
+ if (ZSTD_isError(cBlockSize)) {
3350
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3351
+ return;
3352
+ }
3328
3353
 
3329
3354
  ip += ZSTD_blockHeaderSize;
3330
3355
  remainingSize -= ZSTD_blockHeaderSize;
3331
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3356
+ if (cBlockSize > remainingSize) {
3357
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3358
+ return;
3359
+ }
3332
3360
 
3333
3361
  if (cBlockSize == 0) break; /* bt_end */
3334
3362
 
3335
3363
  ip += cBlockSize;
3336
3364
  remainingSize -= cBlockSize;
3365
+ nbBlocks++;
3337
3366
  }
3338
3367
 
3339
- return ip - (const BYTE*)src;
3368
+ *cSize = ip - (const BYTE*)src;
3369
+ *dBound = nbBlocks * BLOCKSIZE;
3340
3370
  }
3341
3371
 
3342
3372
  /*******************************
@@ -3452,11 +3482,6 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3452
3482
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3453
3483
  }
3454
3484
 
3455
- size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
3456
- {
3457
- return ZSTD_findFrameCompressedSize(src, compressedSize);
3458
- }
3459
-
3460
3485
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3461
3486
  {
3462
3487
  return (ZSTDv02_Dctx*)ZSTD_createDCtx();
@@ -35,13 +35,18 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
35
35
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv02_isError())
43
- */
44
- size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
@@ -402,11 +402,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
402
402
  switch(srcSize)
403
403
  {
404
404
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
405
+ /* fallthrough */
405
406
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
407
+ /* fallthrough */
406
408
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
409
+ /* fallthrough */
407
410
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
411
+ /* fallthrough */
408
412
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
413
+ /* fallthrough */
409
414
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
415
+ /* fallthrough */
410
416
  default:;
411
417
  }
412
418
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -2363,6 +2369,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2363
2369
  #define LITERAL_NOENTROPY 63
2364
2370
  #define COMMAND_NOENTROPY 7 /* to remove */
2365
2371
 
2372
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2373
+
2366
2374
  static const size_t ZSTD_blockHeaderSize = 3;
2367
2375
  static const size_t ZSTD_frameHeaderSize = 4;
2368
2376
 
@@ -2731,7 +2739,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2731
2739
  BYTE* const base, BYTE* const oend)
2732
2740
  {
2733
2741
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
2734
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
2742
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
2735
2743
  const BYTE* const ostart = op;
2736
2744
  BYTE* const oLitEnd = op + sequence.litLength;
2737
2745
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -2947,36 +2955,59 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
2947
2955
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
2948
2956
  }
2949
2957
 
2950
- static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
2958
+ /* ZSTD_errorFrameSizeInfoLegacy() :
2959
+ assumes `cSize` and `dBound` are _not_ NULL */
2960
+ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
2961
+ {
2962
+ *cSize = ret;
2963
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
2964
+ }
2965
+
2966
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
2951
2967
  {
2952
2968
  const BYTE* ip = (const BYTE*)src;
2953
2969
  size_t remainingSize = srcSize;
2970
+ size_t nbBlocks = 0;
2954
2971
  U32 magicNumber;
2955
2972
  blockProperties_t blockProperties;
2956
2973
 
2957
2974
  /* Frame Header */
2958
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
2975
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
2976
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
2977
+ return;
2978
+ }
2959
2979
  magicNumber = MEM_readLE32(src);
2960
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
2980
+ if (magicNumber != ZSTD_magicNumber) {
2981
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
2982
+ return;
2983
+ }
2961
2984
  ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
2962
2985
 
2963
2986
  /* Loop on each block */
2964
2987
  while (1)
2965
2988
  {
2966
2989
  size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
2967
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
2990
+ if (ZSTD_isError(cBlockSize)) {
2991
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
2992
+ return;
2993
+ }
2968
2994
 
2969
2995
  ip += ZSTD_blockHeaderSize;
2970
2996
  remainingSize -= ZSTD_blockHeaderSize;
2971
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
2997
+ if (cBlockSize > remainingSize) {
2998
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
2999
+ return;
3000
+ }
2972
3001
 
2973
3002
  if (cBlockSize == 0) break; /* bt_end */
2974
3003
 
2975
3004
  ip += cBlockSize;
2976
3005
  remainingSize -= cBlockSize;
3006
+ nbBlocks++;
2977
3007
  }
2978
3008
 
2979
- return ip - (const BYTE*)src;
3009
+ *cSize = ip - (const BYTE*)src;
3010
+ *dBound = nbBlocks * BLOCKSIZE;
2980
3011
  }
2981
3012
 
2982
3013
 
@@ -3093,11 +3124,6 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
3093
3124
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3094
3125
  }
3095
3126
 
3096
- size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
3097
- {
3098
- return ZSTD_findFrameCompressedSize(src, srcSize);
3099
- }
3100
-
3101
3127
  ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
3102
3128
  {
3103
3129
  return (ZSTDv03_Dctx*)ZSTD_createDCtx();
@@ -35,13 +35,18 @@ ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format
35
35
  size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv03_isError())
43
- */
44
- size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
@@ -9,45 +9,19 @@
9
9
  */
10
10
 
11
11
 
12
- /*- Dependencies -*/
12
+ /******************************************
13
+ * Includes
14
+ ******************************************/
15
+ #include <stddef.h> /* size_t, ptrdiff_t */
16
+ #include <string.h> /* memcpy */
17
+
13
18
  #include "zstd_v04.h"
14
19
  #include "error_private.h"
15
20
 
16
21
 
17
22
  /* ******************************************************************
18
- mem.h
19
- low-level memory access routines
20
- Copyright (C) 2013-2015, Yann Collet.
21
-
22
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
23
-
24
- Redistribution and use in source and binary forms, with or without
25
- modification, are permitted provided that the following conditions are
26
- met:
27
-
28
- * Redistributions of source code must retain the above copyright
29
- notice, this list of conditions and the following disclaimer.
30
- * Redistributions in binary form must reproduce the above
31
- copyright notice, this list of conditions and the following disclaimer
32
- in the documentation and/or other materials provided with the
33
- distribution.
34
-
35
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46
-
47
- You can contact the author at :
48
- - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
49
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
50
- ****************************************************************** */
23
+ * mem.h
24
+ *******************************************************************/
51
25
  #ifndef MEM_H_MODULE
52
26
  #define MEM_H_MODULE
53
27
 
@@ -55,12 +29,6 @@
55
29
  extern "C" {
56
30
  #endif
57
31
 
58
- /******************************************
59
- * Includes
60
- ******************************************/
61
- #include <stddef.h> /* size_t, ptrdiff_t */
62
- #include <string.h> /* memcpy */
63
-
64
32
 
65
33
  /******************************************
66
34
  * Compiler-specific
@@ -103,6 +71,15 @@ extern "C" {
103
71
  #endif
104
72
 
105
73
 
74
+ /*-*************************************
75
+ * Debug
76
+ ***************************************/
77
+ #include "debug.h"
78
+ #ifndef assert
79
+ # define assert(condition) ((void)0)
80
+ #endif
81
+
82
+
106
83
  /****************************************************************
107
84
  * Memory I/O
108
85
  *****************************************************************/
@@ -255,61 +232,15 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
255
232
  /*
256
233
  zstd - standard compression library
257
234
  Header File for static linking only
258
- Copyright (C) 2014-2015, Yann Collet.
259
-
260
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
261
-
262
- Redistribution and use in source and binary forms, with or without
263
- modification, are permitted provided that the following conditions are
264
- met:
265
- * Redistributions of source code must retain the above copyright
266
- notice, this list of conditions and the following disclaimer.
267
- * Redistributions in binary form must reproduce the above
268
- copyright notice, this list of conditions and the following disclaimer
269
- in the documentation and/or other materials provided with the
270
- distribution.
271
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
272
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
273
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
274
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
275
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
276
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
277
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
278
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
279
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
280
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
281
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
282
-
283
- You can contact the author at :
284
- - zstd source repository : https://github.com/Cyan4973/zstd
285
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
286
235
  */
287
236
  #ifndef ZSTD_STATIC_H
288
237
  #define ZSTD_STATIC_H
289
238
 
290
- /* The objects defined into this file shall be considered experimental.
291
- * They are not considered stable, as their prototype may change in the future.
292
- * You can use them for tests, provide feedback, or if you can endure risks of future changes.
293
- */
294
-
295
- #if defined (__cplusplus)
296
- extern "C" {
297
- #endif
298
239
 
299
240
  /* *************************************
300
241
  * Types
301
242
  ***************************************/
302
- #define ZSTD_WINDOWLOG_MAX 26
303
- #define ZSTD_WINDOWLOG_MIN 18
304
243
  #define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
305
- #define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
306
- #define ZSTD_CONTENTLOG_MIN 4
307
- #define ZSTD_HASHLOG_MAX 28
308
- #define ZSTD_HASHLOG_MIN 4
309
- #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
310
- #define ZSTD_SEARCHLOG_MIN 1
311
- #define ZSTD_SEARCHLENGTH_MAX 7
312
- #define ZSTD_SEARCHLENGTH_MIN 4
313
244
 
314
245
  /** from faster to stronger */
315
246
  typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
@@ -381,9 +312,6 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstS
381
312
  */
382
313
 
383
314
 
384
- #if defined (__cplusplus)
385
- }
386
- #endif
387
315
 
388
316
 
389
317
  #endif /* ZSTD_STATIC_H */
@@ -392,42 +320,10 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstS
392
320
  /*
393
321
  zstd_internal - common functions to include
394
322
  Header File for include
395
- Copyright (C) 2014-2015, Yann Collet.
396
-
397
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
398
-
399
- Redistribution and use in source and binary forms, with or without
400
- modification, are permitted provided that the following conditions are
401
- met:
402
- * Redistributions of source code must retain the above copyright
403
- notice, this list of conditions and the following disclaimer.
404
- * Redistributions in binary form must reproduce the above
405
- copyright notice, this list of conditions and the following disclaimer
406
- in the documentation and/or other materials provided with the
407
- distribution.
408
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
409
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
410
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
411
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
412
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
413
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
414
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
415
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
416
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
417
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
418
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
419
-
420
- You can contact the author at :
421
- - zstd source repository : https://github.com/Cyan4973/zstd
422
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
423
323
  */
424
324
  #ifndef ZSTD_CCOMMON_H_MODULE
425
325
  #define ZSTD_CCOMMON_H_MODULE
426
326
 
427
- #if defined (__cplusplus)
428
- extern "C" {
429
- #endif
430
-
431
327
  /* *************************************
432
328
  * Common macros
433
329
  ***************************************/
@@ -477,6 +373,8 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
477
373
  #define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
478
374
  #define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
479
375
 
376
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
377
+
480
378
  typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
481
379
 
482
380
 
@@ -499,44 +397,10 @@ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
499
397
  }
500
398
 
501
399
 
502
- #if defined (__cplusplus)
503
- }
504
- #endif
505
-
506
400
 
507
401
  /* ******************************************************************
508
402
  FSE : Finite State Entropy coder
509
403
  header file
510
- Copyright (C) 2013-2015, Yann Collet.
511
-
512
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
513
-
514
- Redistribution and use in source and binary forms, with or without
515
- modification, are permitted provided that the following conditions are
516
- met:
517
-
518
- * Redistributions of source code must retain the above copyright
519
- notice, this list of conditions and the following disclaimer.
520
- * Redistributions in binary form must reproduce the above
521
- copyright notice, this list of conditions and the following disclaimer
522
- in the documentation and/or other materials provided with the
523
- distribution.
524
-
525
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
526
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
527
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
528
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
529
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
530
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
531
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
532
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
533
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
534
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
535
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
536
-
537
- You can contact the author at :
538
- - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
539
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
540
404
  ****************************************************************** */
541
405
  #ifndef FSE_H
542
406
  #define FSE_H
@@ -1221,6 +1085,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
1221
1085
  if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1222
1086
 
1223
1087
  /* Init, lay down lowprob symbols */
1088
+ memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1224
1089
  DTableH.tableLog = (U16)tableLog;
1225
1090
  for (s=0; s<=maxSymbolValue; s++)
1226
1091
  {
@@ -2997,7 +2862,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2997
2862
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
2998
2863
  {
2999
2864
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3000
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
2865
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3001
2866
  BYTE* const oLitEnd = op + sequence.litLength;
3002
2867
  const size_t sequenceLength = sequence.litLength + sequence.matchLength;
3003
2868
  BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
@@ -3070,7 +2935,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3070
2935
  }
3071
2936
  else
3072
2937
  {
3073
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
2938
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8, but must be signed */
3074
2939
  }
3075
2940
  return sequenceLength;
3076
2941
  }
@@ -3256,34 +3121,57 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
3256
3121
  return op-ostart;
3257
3122
  }
3258
3123
 
3259
- static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
3124
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3125
+ assumes `cSize` and `dBound` are _not_ NULL */
3126
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3127
+ {
3128
+ *cSize = ret;
3129
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3130
+ }
3131
+
3132
+ void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3260
3133
  {
3261
3134
  const BYTE* ip = (const BYTE*)src;
3262
3135
  size_t remainingSize = srcSize;
3136
+ size_t nbBlocks = 0;
3263
3137
  blockProperties_t blockProperties;
3264
3138
 
3265
3139
  /* Frame Header */
3266
- if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
3267
- if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
3140
+ if (srcSize < ZSTD_frameHeaderSize_min) {
3141
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3142
+ return;
3143
+ }
3144
+ if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
3145
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3146
+ return;
3147
+ }
3268
3148
  ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
3269
3149
 
3270
3150
  /* Loop on each block */
3271
3151
  while (1)
3272
3152
  {
3273
3153
  size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3274
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
3154
+ if (ZSTD_isError(cBlockSize)) {
3155
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3156
+ return;
3157
+ }
3275
3158
 
3276
3159
  ip += ZSTD_blockHeaderSize;
3277
3160
  remainingSize -= ZSTD_blockHeaderSize;
3278
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3161
+ if (cBlockSize > remainingSize) {
3162
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3163
+ return;
3164
+ }
3279
3165
 
3280
3166
  if (cBlockSize == 0) break; /* bt_end */
3281
3167
 
3282
3168
  ip += cBlockSize;
3283
3169
  remainingSize -= cBlockSize;
3170
+ nbBlocks++;
3284
3171
  }
3285
3172
 
3286
- return ip - (const BYTE*)src;
3173
+ *cSize = ip - (const BYTE*)src;
3174
+ *dBound = nbBlocks * BLOCKSIZE;
3287
3175
  }
3288
3176
 
3289
3177
  /* ******************************
@@ -3528,12 +3416,14 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3528
3416
  char* const oend = ostart + *maxDstSizePtr;
3529
3417
  U32 notDone = 1;
3530
3418
 
3419
+ DEBUGLOG(5, "ZBUFF_decompressContinue");
3531
3420
  while (notDone)
3532
3421
  {
3533
3422
  switch(zbc->stage)
3534
3423
  {
3535
3424
 
3536
3425
  case ZBUFFds_init :
3426
+ DEBUGLOG(5, "ZBUFF_decompressContinue: stage==ZBUFFds_init => ERROR(init_missing)");
3537
3427
  return ERROR(init_missing);
3538
3428
 
3539
3429
  case ZBUFFds_readHeader :
@@ -3713,11 +3603,6 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3713
3603
  #endif
3714
3604
  }
3715
3605
 
3716
- size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
3717
- {
3718
- return ZSTD_findFrameCompressedSize(src, srcSize);
3719
- }
3720
-
3721
3606
  size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
3722
3607
 
3723
3608
  size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx)
@@ -3733,7 +3618,7 @@ size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSi
3733
3618
 
3734
3619
 
3735
3620
  ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void) { return ZBUFF_createDCtx(); }
3736
- size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx) { return ZBUFF_freeDCtx(dctx); }
3621
+ size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx) { return ZBUFF_freeDCtx(dctx); }
3737
3622
 
3738
3623
  size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx) { return ZBUFF_decompressInit(dctx); }
3739
3624
  size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* src, size_t srcSize)
@@ -3741,13 +3626,9 @@ size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* src, s
3741
3626
 
3742
3627
  size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
3743
3628
  {
3629
+ DEBUGLOG(5, "ZBUFFv04_decompressContinue");
3744
3630
  return ZBUFF_decompressContinue(dctx, dst, maxDstSizePtr, src, srcSizePtr);
3745
3631
  }
3746
3632
 
3747
3633
  ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); }
3748
3634
  size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); }
3749
-
3750
- size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
3751
- {
3752
- return ZSTD_getFrameParams(params, src, srcSize);
3753
- }