extzstd 0.2 → 0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +13 -0
  3. data/README.md +17 -14
  4. data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/Makefile +99 -53
  7. data/contrib/zstd/README.md +59 -39
  8. data/contrib/zstd/TESTING.md +1 -1
  9. data/contrib/zstd/appveyor.yml +17 -6
  10. data/contrib/zstd/lib/BUCK +29 -2
  11. data/contrib/zstd/lib/Makefile +118 -21
  12. data/contrib/zstd/lib/README.md +84 -44
  13. data/contrib/zstd/lib/common/bitstream.h +17 -33
  14. data/contrib/zstd/lib/common/compiler.h +62 -8
  15. data/contrib/zstd/lib/common/cpu.h +215 -0
  16. data/contrib/zstd/lib/common/debug.c +44 -0
  17. data/contrib/zstd/lib/common/debug.h +134 -0
  18. data/contrib/zstd/lib/common/entropy_common.c +16 -1
  19. data/contrib/zstd/lib/common/error_private.c +7 -0
  20. data/contrib/zstd/lib/common/fse.h +48 -44
  21. data/contrib/zstd/lib/common/fse_decompress.c +3 -3
  22. data/contrib/zstd/lib/common/huf.h +169 -113
  23. data/contrib/zstd/lib/common/mem.h +20 -2
  24. data/contrib/zstd/lib/common/pool.c +135 -49
  25. data/contrib/zstd/lib/common/pool.h +40 -21
  26. data/contrib/zstd/lib/common/threading.c +2 -2
  27. data/contrib/zstd/lib/common/threading.h +12 -12
  28. data/contrib/zstd/lib/common/xxhash.c +3 -2
  29. data/contrib/zstd/lib/common/zstd_common.c +3 -6
  30. data/contrib/zstd/lib/common/zstd_errors.h +17 -7
  31. data/contrib/zstd/lib/common/zstd_internal.h +76 -48
  32. data/contrib/zstd/lib/compress/fse_compress.c +89 -209
  33. data/contrib/zstd/lib/compress/hist.c +203 -0
  34. data/contrib/zstd/lib/compress/hist.h +95 -0
  35. data/contrib/zstd/lib/compress/huf_compress.c +188 -80
  36. data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
  37. data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
  38. data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
  39. data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
  40. data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
  41. data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
  46. data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
  47. data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
  48. data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
  49. data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
  50. data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
  51. data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
  52. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  53. data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
  55. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  56. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
  57. data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
  58. data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
  59. data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
  60. data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
  61. data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
  62. data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
  63. data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
  64. data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
  65. data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
  66. data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
  67. data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
  68. data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
  69. data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
  70. data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
  71. data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
  72. data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
  73. data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
  74. data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
  75. data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
  76. data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
  77. data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
  78. data/contrib/zstd/lib/zstd.h +1346 -832
  79. data/ext/extzstd.c +27 -19
  80. data/ext/extzstd_stream.c +20 -4
  81. data/ext/zstd_compress.c +1 -0
  82. data/ext/zstd_decompress.c +4 -0
  83. data/ext/zstd_dictbuilder.c +4 -0
  84. data/ext/zstd_dictbuilder_fastcover.c +5 -0
  85. data/lib/extzstd.rb +52 -220
  86. data/lib/extzstd/version.rb +1 -1
  87. metadata +21 -7
  88. data/contrib/zstd/circle.yml +0 -63
@@ -35,13 +35,18 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
35
35
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv01_isError())
43
- */
44
- size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
@@ -399,11 +399,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
399
399
  switch(srcSize)
400
400
  {
401
401
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
402
+ /* fallthrough */
402
403
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
404
+ /* fallthrough */
403
405
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
406
+ /* fallthrough */
404
407
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
408
+ /* fallthrough */
405
409
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
410
+ /* fallthrough */
406
411
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
412
+ /* fallthrough */
407
413
  default:;
408
414
  }
409
415
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -2722,6 +2728,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2722
2728
  #define LITERAL_NOENTROPY 63
2723
2729
  #define COMMAND_NOENTROPY 7 /* to remove */
2724
2730
 
2731
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2732
+
2725
2733
  static const size_t ZSTD_blockHeaderSize = 3;
2726
2734
  static const size_t ZSTD_frameHeaderSize = 4;
2727
2735
 
@@ -3090,7 +3098,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3090
3098
  BYTE* const base, BYTE* const oend)
3091
3099
  {
3092
3100
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
3093
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
3101
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
3094
3102
  const BYTE* const ostart = op;
3095
3103
  BYTE* const oLitEnd = op + sequence.litLength;
3096
3104
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -3306,37 +3314,59 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3306
3314
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3307
3315
  }
3308
3316
 
3309
- static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
3317
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3318
+ assumes `cSize` and `dBound` are _not_ NULL */
3319
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3310
3320
  {
3321
+ *cSize = ret;
3322
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3323
+ }
3311
3324
 
3325
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3326
+ {
3312
3327
  const BYTE* ip = (const BYTE*)src;
3313
3328
  size_t remainingSize = srcSize;
3329
+ size_t nbBlocks = 0;
3314
3330
  U32 magicNumber;
3315
3331
  blockProperties_t blockProperties;
3316
3332
 
3317
3333
  /* Frame Header */
3318
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
3334
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
3335
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3336
+ return;
3337
+ }
3319
3338
  magicNumber = MEM_readLE32(src);
3320
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
3339
+ if (magicNumber != ZSTD_magicNumber) {
3340
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3341
+ return;
3342
+ }
3321
3343
  ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3322
3344
 
3323
3345
  /* Loop on each block */
3324
3346
  while (1)
3325
3347
  {
3326
3348
  size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3327
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
3349
+ if (ZSTD_isError(cBlockSize)) {
3350
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3351
+ return;
3352
+ }
3328
3353
 
3329
3354
  ip += ZSTD_blockHeaderSize;
3330
3355
  remainingSize -= ZSTD_blockHeaderSize;
3331
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3356
+ if (cBlockSize > remainingSize) {
3357
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3358
+ return;
3359
+ }
3332
3360
 
3333
3361
  if (cBlockSize == 0) break; /* bt_end */
3334
3362
 
3335
3363
  ip += cBlockSize;
3336
3364
  remainingSize -= cBlockSize;
3365
+ nbBlocks++;
3337
3366
  }
3338
3367
 
3339
- return ip - (const BYTE*)src;
3368
+ *cSize = ip - (const BYTE*)src;
3369
+ *dBound = nbBlocks * BLOCKSIZE;
3340
3370
  }
3341
3371
 
3342
3372
  /*******************************
@@ -3452,11 +3482,6 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3452
3482
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3453
3483
  }
3454
3484
 
3455
- size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
3456
- {
3457
- return ZSTD_findFrameCompressedSize(src, compressedSize);
3458
- }
3459
-
3460
3485
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3461
3486
  {
3462
3487
  return (ZSTDv02_Dctx*)ZSTD_createDCtx();
@@ -35,13 +35,18 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
35
35
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv02_isError())
43
- */
44
- size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
@@ -402,11 +402,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
402
402
  switch(srcSize)
403
403
  {
404
404
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
405
+ /* fallthrough */
405
406
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
407
+ /* fallthrough */
406
408
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
409
+ /* fallthrough */
407
410
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
411
+ /* fallthrough */
408
412
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
413
+ /* fallthrough */
409
414
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
415
+ /* fallthrough */
410
416
  default:;
411
417
  }
412
418
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -2363,6 +2369,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2363
2369
  #define LITERAL_NOENTROPY 63
2364
2370
  #define COMMAND_NOENTROPY 7 /* to remove */
2365
2371
 
2372
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2373
+
2366
2374
  static const size_t ZSTD_blockHeaderSize = 3;
2367
2375
  static const size_t ZSTD_frameHeaderSize = 4;
2368
2376
 
@@ -2731,7 +2739,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2731
2739
  BYTE* const base, BYTE* const oend)
2732
2740
  {
2733
2741
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
2734
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
2742
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
2735
2743
  const BYTE* const ostart = op;
2736
2744
  BYTE* const oLitEnd = op + sequence.litLength;
2737
2745
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -2947,36 +2955,59 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
2947
2955
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
2948
2956
  }
2949
2957
 
2950
- static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
2958
+ /* ZSTD_errorFrameSizeInfoLegacy() :
2959
+ assumes `cSize` and `dBound` are _not_ NULL */
2960
+ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
2961
+ {
2962
+ *cSize = ret;
2963
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
2964
+ }
2965
+
2966
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
2951
2967
  {
2952
2968
  const BYTE* ip = (const BYTE*)src;
2953
2969
  size_t remainingSize = srcSize;
2970
+ size_t nbBlocks = 0;
2954
2971
  U32 magicNumber;
2955
2972
  blockProperties_t blockProperties;
2956
2973
 
2957
2974
  /* Frame Header */
2958
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
2975
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
2976
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
2977
+ return;
2978
+ }
2959
2979
  magicNumber = MEM_readLE32(src);
2960
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
2980
+ if (magicNumber != ZSTD_magicNumber) {
2981
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
2982
+ return;
2983
+ }
2961
2984
  ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
2962
2985
 
2963
2986
  /* Loop on each block */
2964
2987
  while (1)
2965
2988
  {
2966
2989
  size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
2967
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
2990
+ if (ZSTD_isError(cBlockSize)) {
2991
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
2992
+ return;
2993
+ }
2968
2994
 
2969
2995
  ip += ZSTD_blockHeaderSize;
2970
2996
  remainingSize -= ZSTD_blockHeaderSize;
2971
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
2997
+ if (cBlockSize > remainingSize) {
2998
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
2999
+ return;
3000
+ }
2972
3001
 
2973
3002
  if (cBlockSize == 0) break; /* bt_end */
2974
3003
 
2975
3004
  ip += cBlockSize;
2976
3005
  remainingSize -= cBlockSize;
3006
+ nbBlocks++;
2977
3007
  }
2978
3008
 
2979
- return ip - (const BYTE*)src;
3009
+ *cSize = ip - (const BYTE*)src;
3010
+ *dBound = nbBlocks * BLOCKSIZE;
2980
3011
  }
2981
3012
 
2982
3013
 
@@ -3093,11 +3124,6 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
3093
3124
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3094
3125
  }
3095
3126
 
3096
- size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
3097
- {
3098
- return ZSTD_findFrameCompressedSize(src, srcSize);
3099
- }
3100
-
3101
3127
  ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
3102
3128
  {
3103
3129
  return (ZSTDv03_Dctx*)ZSTD_createDCtx();
@@ -35,13 +35,18 @@ ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format
35
35
  size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
36
36
  const void* src, size_t compressedSize);
37
37
 
38
- /**
39
- ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
40
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
- return : the number of bytes that would be read to decompress this frame
42
- or an errorCode if it fails (which can be tested using ZSTDv03_isError())
43
- */
44
- size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
38
+ /**
39
+ ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
45
50
 
46
51
  /**
47
52
  ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
@@ -9,45 +9,19 @@
9
9
  */
10
10
 
11
11
 
12
- /*- Dependencies -*/
12
+ /******************************************
13
+ * Includes
14
+ ******************************************/
15
+ #include <stddef.h> /* size_t, ptrdiff_t */
16
+ #include <string.h> /* memcpy */
17
+
13
18
  #include "zstd_v04.h"
14
19
  #include "error_private.h"
15
20
 
16
21
 
17
22
  /* ******************************************************************
18
- mem.h
19
- low-level memory access routines
20
- Copyright (C) 2013-2015, Yann Collet.
21
-
22
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
23
-
24
- Redistribution and use in source and binary forms, with or without
25
- modification, are permitted provided that the following conditions are
26
- met:
27
-
28
- * Redistributions of source code must retain the above copyright
29
- notice, this list of conditions and the following disclaimer.
30
- * Redistributions in binary form must reproduce the above
31
- copyright notice, this list of conditions and the following disclaimer
32
- in the documentation and/or other materials provided with the
33
- distribution.
34
-
35
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46
-
47
- You can contact the author at :
48
- - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
49
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
50
- ****************************************************************** */
23
+ * mem.h
24
+ *******************************************************************/
51
25
  #ifndef MEM_H_MODULE
52
26
  #define MEM_H_MODULE
53
27
 
@@ -55,12 +29,6 @@
55
29
  extern "C" {
56
30
  #endif
57
31
 
58
- /******************************************
59
- * Includes
60
- ******************************************/
61
- #include <stddef.h> /* size_t, ptrdiff_t */
62
- #include <string.h> /* memcpy */
63
-
64
32
 
65
33
  /******************************************
66
34
  * Compiler-specific
@@ -103,6 +71,15 @@ extern "C" {
103
71
  #endif
104
72
 
105
73
 
74
+ /*-*************************************
75
+ * Debug
76
+ ***************************************/
77
+ #include "debug.h"
78
+ #ifndef assert
79
+ # define assert(condition) ((void)0)
80
+ #endif
81
+
82
+
106
83
  /****************************************************************
107
84
  * Memory I/O
108
85
  *****************************************************************/
@@ -255,61 +232,15 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
255
232
  /*
256
233
  zstd - standard compression library
257
234
  Header File for static linking only
258
- Copyright (C) 2014-2015, Yann Collet.
259
-
260
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
261
-
262
- Redistribution and use in source and binary forms, with or without
263
- modification, are permitted provided that the following conditions are
264
- met:
265
- * Redistributions of source code must retain the above copyright
266
- notice, this list of conditions and the following disclaimer.
267
- * Redistributions in binary form must reproduce the above
268
- copyright notice, this list of conditions and the following disclaimer
269
- in the documentation and/or other materials provided with the
270
- distribution.
271
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
272
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
273
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
274
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
275
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
276
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
277
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
278
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
279
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
280
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
281
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
282
-
283
- You can contact the author at :
284
- - zstd source repository : https://github.com/Cyan4973/zstd
285
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
286
235
  */
287
236
  #ifndef ZSTD_STATIC_H
288
237
  #define ZSTD_STATIC_H
289
238
 
290
- /* The objects defined into this file shall be considered experimental.
291
- * They are not considered stable, as their prototype may change in the future.
292
- * You can use them for tests, provide feedback, or if you can endure risks of future changes.
293
- */
294
-
295
- #if defined (__cplusplus)
296
- extern "C" {
297
- #endif
298
239
 
299
240
  /* *************************************
300
241
  * Types
301
242
  ***************************************/
302
- #define ZSTD_WINDOWLOG_MAX 26
303
- #define ZSTD_WINDOWLOG_MIN 18
304
243
  #define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
305
- #define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
306
- #define ZSTD_CONTENTLOG_MIN 4
307
- #define ZSTD_HASHLOG_MAX 28
308
- #define ZSTD_HASHLOG_MIN 4
309
- #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
310
- #define ZSTD_SEARCHLOG_MIN 1
311
- #define ZSTD_SEARCHLENGTH_MAX 7
312
- #define ZSTD_SEARCHLENGTH_MIN 4
313
244
 
314
245
  /** from faster to stronger */
315
246
  typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
@@ -381,9 +312,6 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstS
381
312
  */
382
313
 
383
314
 
384
- #if defined (__cplusplus)
385
- }
386
- #endif
387
315
 
388
316
 
389
317
  #endif /* ZSTD_STATIC_H */
@@ -392,42 +320,10 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstS
392
320
  /*
393
321
  zstd_internal - common functions to include
394
322
  Header File for include
395
- Copyright (C) 2014-2015, Yann Collet.
396
-
397
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
398
-
399
- Redistribution and use in source and binary forms, with or without
400
- modification, are permitted provided that the following conditions are
401
- met:
402
- * Redistributions of source code must retain the above copyright
403
- notice, this list of conditions and the following disclaimer.
404
- * Redistributions in binary form must reproduce the above
405
- copyright notice, this list of conditions and the following disclaimer
406
- in the documentation and/or other materials provided with the
407
- distribution.
408
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
409
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
410
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
411
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
412
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
413
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
414
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
415
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
416
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
417
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
418
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
419
-
420
- You can contact the author at :
421
- - zstd source repository : https://github.com/Cyan4973/zstd
422
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
423
323
  */
424
324
  #ifndef ZSTD_CCOMMON_H_MODULE
425
325
  #define ZSTD_CCOMMON_H_MODULE
426
326
 
427
- #if defined (__cplusplus)
428
- extern "C" {
429
- #endif
430
-
431
327
  /* *************************************
432
328
  * Common macros
433
329
  ***************************************/
@@ -477,6 +373,8 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
477
373
  #define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
478
374
  #define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
479
375
 
376
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
377
+
480
378
  typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
481
379
 
482
380
 
@@ -499,44 +397,10 @@ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
499
397
  }
500
398
 
501
399
 
502
- #if defined (__cplusplus)
503
- }
504
- #endif
505
-
506
400
 
507
401
  /* ******************************************************************
508
402
  FSE : Finite State Entropy coder
509
403
  header file
510
- Copyright (C) 2013-2015, Yann Collet.
511
-
512
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
513
-
514
- Redistribution and use in source and binary forms, with or without
515
- modification, are permitted provided that the following conditions are
516
- met:
517
-
518
- * Redistributions of source code must retain the above copyright
519
- notice, this list of conditions and the following disclaimer.
520
- * Redistributions in binary form must reproduce the above
521
- copyright notice, this list of conditions and the following disclaimer
522
- in the documentation and/or other materials provided with the
523
- distribution.
524
-
525
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
526
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
527
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
528
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
529
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
530
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
531
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
532
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
533
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
534
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
535
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
536
-
537
- You can contact the author at :
538
- - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
539
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
540
404
  ****************************************************************** */
541
405
  #ifndef FSE_H
542
406
  #define FSE_H
@@ -1221,6 +1085,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
1221
1085
  if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1222
1086
 
1223
1087
  /* Init, lay down lowprob symbols */
1088
+ memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1224
1089
  DTableH.tableLog = (U16)tableLog;
1225
1090
  for (s=0; s<=maxSymbolValue; s++)
1226
1091
  {
@@ -2997,7 +2862,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2997
2862
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
2998
2863
  {
2999
2864
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3000
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
2865
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3001
2866
  BYTE* const oLitEnd = op + sequence.litLength;
3002
2867
  const size_t sequenceLength = sequence.litLength + sequence.matchLength;
3003
2868
  BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
@@ -3070,7 +2935,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3070
2935
  }
3071
2936
  else
3072
2937
  {
3073
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
2938
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8, but must be signed */
3074
2939
  }
3075
2940
  return sequenceLength;
3076
2941
  }
@@ -3256,34 +3121,57 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
3256
3121
  return op-ostart;
3257
3122
  }
3258
3123
 
3259
- static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
3124
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3125
+ assumes `cSize` and `dBound` are _not_ NULL */
3126
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3127
+ {
3128
+ *cSize = ret;
3129
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3130
+ }
3131
+
3132
+ void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3260
3133
  {
3261
3134
  const BYTE* ip = (const BYTE*)src;
3262
3135
  size_t remainingSize = srcSize;
3136
+ size_t nbBlocks = 0;
3263
3137
  blockProperties_t blockProperties;
3264
3138
 
3265
3139
  /* Frame Header */
3266
- if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
3267
- if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
3140
+ if (srcSize < ZSTD_frameHeaderSize_min) {
3141
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3142
+ return;
3143
+ }
3144
+ if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
3145
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3146
+ return;
3147
+ }
3268
3148
  ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
3269
3149
 
3270
3150
  /* Loop on each block */
3271
3151
  while (1)
3272
3152
  {
3273
3153
  size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3274
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
3154
+ if (ZSTD_isError(cBlockSize)) {
3155
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3156
+ return;
3157
+ }
3275
3158
 
3276
3159
  ip += ZSTD_blockHeaderSize;
3277
3160
  remainingSize -= ZSTD_blockHeaderSize;
3278
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3161
+ if (cBlockSize > remainingSize) {
3162
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3163
+ return;
3164
+ }
3279
3165
 
3280
3166
  if (cBlockSize == 0) break; /* bt_end */
3281
3167
 
3282
3168
  ip += cBlockSize;
3283
3169
  remainingSize -= cBlockSize;
3170
+ nbBlocks++;
3284
3171
  }
3285
3172
 
3286
- return ip - (const BYTE*)src;
3173
+ *cSize = ip - (const BYTE*)src;
3174
+ *dBound = nbBlocks * BLOCKSIZE;
3287
3175
  }
3288
3176
 
3289
3177
  /* ******************************
@@ -3528,12 +3416,14 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3528
3416
  char* const oend = ostart + *maxDstSizePtr;
3529
3417
  U32 notDone = 1;
3530
3418
 
3419
+ DEBUGLOG(5, "ZBUFF_decompressContinue");
3531
3420
  while (notDone)
3532
3421
  {
3533
3422
  switch(zbc->stage)
3534
3423
  {
3535
3424
 
3536
3425
  case ZBUFFds_init :
3426
+ DEBUGLOG(5, "ZBUFF_decompressContinue: stage==ZBUFFds_init => ERROR(init_missing)");
3537
3427
  return ERROR(init_missing);
3538
3428
 
3539
3429
  case ZBUFFds_readHeader :
@@ -3713,11 +3603,6 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3713
3603
  #endif
3714
3604
  }
3715
3605
 
3716
- size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
3717
- {
3718
- return ZSTD_findFrameCompressedSize(src, srcSize);
3719
- }
3720
-
3721
3606
  size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
3722
3607
 
3723
3608
  size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx)
@@ -3733,7 +3618,7 @@ size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSi
3733
3618
 
3734
3619
 
3735
3620
  ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void) { return ZBUFF_createDCtx(); }
3736
- size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx) { return ZBUFF_freeDCtx(dctx); }
3621
+ size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx) { return ZBUFF_freeDCtx(dctx); }
3737
3622
 
3738
3623
  size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx) { return ZBUFF_decompressInit(dctx); }
3739
3624
  size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* src, size_t srcSize)
@@ -3741,13 +3626,9 @@ size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* src, s
3741
3626
 
3742
3627
  size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
3743
3628
  {
3629
+ DEBUGLOG(5, "ZBUFFv04_decompressContinue");
3744
3630
  return ZBUFF_decompressContinue(dctx, dst, maxDstSizePtr, src, srcSizePtr);
3745
3631
  }
3746
3632
 
3747
3633
  ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); }
3748
3634
  size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); }
3749
-
3750
- size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
3751
- {
3752
- return ZSTD_getFrameParams(params, src, srcSize);
3753
- }