extzstd 0.1 → 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja +5 -0
  3. data/README.md +5 -5
  4. data/contrib/zstd/CONTRIBUTING.md +42 -0
  5. data/contrib/zstd/LICENSE-examples +11 -0
  6. data/contrib/zstd/Makefile +315 -0
  7. data/contrib/zstd/NEWS +261 -0
  8. data/contrib/zstd/PATENTS +33 -0
  9. data/contrib/zstd/README.md +121 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +178 -0
  12. data/contrib/zstd/circle.yml +75 -0
  13. data/contrib/zstd/lib/BUCK +186 -0
  14. data/contrib/zstd/lib/Makefile +163 -0
  15. data/contrib/zstd/lib/README.md +77 -0
  16. data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
  17. data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
  18. data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
  19. data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
  20. data/contrib/zstd/{common → lib/common}/fse.h +94 -34
  21. data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
  22. data/contrib/zstd/{common → lib/common}/huf.h +52 -20
  23. data/contrib/zstd/{common → lib/common}/mem.h +17 -13
  24. data/contrib/zstd/lib/common/pool.c +194 -0
  25. data/contrib/zstd/lib/common/pool.h +56 -0
  26. data/contrib/zstd/lib/common/threading.c +80 -0
  27. data/contrib/zstd/lib/common/threading.h +104 -0
  28. data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
  29. data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
  30. data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
  31. data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
  32. data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
  33. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
  34. data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
  35. data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
  36. data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
  37. data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
  38. data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
  39. data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
  40. data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
  41. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
  42. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  43. data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
  44. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
  45. data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
  46. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
  47. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  48. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
  49. data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
  50. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
  51. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
  52. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
  53. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
  54. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
  55. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
  56. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
  57. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
  58. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
  59. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
  60. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
  61. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
  62. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
  63. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
  64. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
  65. data/contrib/zstd/lib/libzstd.pc.in +14 -0
  66. data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
  67. data/ext/depend +2 -0
  68. data/ext/extconf.rb +4 -4
  69. data/ext/extzstd.c +1 -1
  70. data/ext/zstd_common.c +5 -5
  71. data/ext/zstd_compress.c +3 -3
  72. data/ext/zstd_decompress.c +2 -2
  73. data/ext/zstd_dictbuilder.c +2 -2
  74. data/ext/zstd_legacy_v01.c +1 -1
  75. data/ext/zstd_legacy_v02.c +1 -1
  76. data/ext/zstd_legacy_v03.c +1 -1
  77. data/ext/zstd_legacy_v04.c +1 -1
  78. data/ext/zstd_legacy_v05.c +1 -1
  79. data/ext/zstd_legacy_v06.c +1 -1
  80. data/ext/zstd_legacy_v07.c +1 -1
  81. data/gemstub.rb +9 -5
  82. data/lib/extzstd/version.rb +1 -1
  83. metadata +73 -51
  84. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  85. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  86. data/contrib/zstd/dictBuilder/zdict.h +0 -111
@@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
34
34
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
+ /**
38
+ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv01_isError())
42
+ */
43
+ size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
37
45
  /**
38
46
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
39
47
  */
@@ -2808,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2808
2808
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2809
2809
 
2810
2810
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2811
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2811
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2812
2812
  {
2813
2813
  const BYTE* ip = (const BYTE*)src;
2814
2814
  BYTE* op = (BYTE*)dst;
@@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s
2868
2868
  blockType_t bType;
2869
2869
  U32 phase;
2870
2870
  const BYTE* litPtr;
2871
- size_t litBufSize;
2872
2871
  size_t litSize;
2873
2872
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2874
2873
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2940
2939
  size_t litSize = BLOCKSIZE;
2941
2940
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2942
2941
  dctx->litPtr = dctx->litBuffer;
2943
- dctx->litBufSize = BLOCKSIZE;
2944
2942
  dctx->litSize = litSize;
2943
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2945
2944
  return readSize; /* works if it's an error too */
2946
2945
  }
2947
2946
  case IS_RAW:
@@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2952
2951
  if (litSize > srcSize-3) return ERROR(corruption_detected);
2953
2952
  memcpy(dctx->litBuffer, istart, litSize);
2954
2953
  dctx->litPtr = dctx->litBuffer;
2955
- dctx->litBufSize = BLOCKSIZE;
2956
2954
  dctx->litSize = litSize;
2955
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2957
2956
  return litSize+3;
2958
2957
  }
2959
2958
  /* direct reference into compressed stream */
2960
2959
  dctx->litPtr = istart+3;
2961
- dctx->litBufSize = srcSize-3;
2962
2960
  dctx->litSize = litSize;
2963
2961
  return litSize+3;
2964
2962
  }
@@ -2966,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2966
2964
  {
2967
2965
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2968
2966
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2969
- memset(dctx->litBuffer, istart[3], litSize);
2967
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2970
2968
  dctx->litPtr = dctx->litBuffer;
2971
- dctx->litBufSize = BLOCKSIZE;
2972
2969
  dctx->litSize = litSize;
2973
2970
  return 4;
2974
2971
  }
@@ -3175,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3175
3172
  /* checks */
3176
3173
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3177
3174
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3178
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3175
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3179
3176
 
3180
3177
  /* copy Literals */
3181
3178
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3209,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3209
3206
  }
3210
3207
  op += 8; match += 8;
3211
3208
 
3212
- if (oMatchEnd > oend-12)
3209
+ if (oMatchEnd > oend-(16-MINMATCH))
3213
3210
  {
3214
3211
  if (op < oend_8)
3215
3212
  {
@@ -3221,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3221
3218
  }
3222
3219
  else
3223
3220
  {
3224
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3221
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3225
3222
  }
3226
3223
  }
3227
3224
 
@@ -3241,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
3241
3238
  BYTE* const oend = ostart + maxDstSize;
3242
3239
  size_t errorCode, dumpsLength;
3243
3240
  const BYTE* litPtr = dctx->litPtr;
3244
- const BYTE* const litMax = litPtr + dctx->litBufSize;
3245
3241
  const BYTE* const litEnd = litPtr + dctx->litSize;
3246
3242
  int nbSeq;
3247
3243
  const BYTE* dumps;
@@ -3277,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
3277
3273
  size_t oneSeqSize;
3278
3274
  nbSeq--;
3279
3275
  ZSTD_decodeSequence(&sequence, &seqState);
3280
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
3276
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
3281
3277
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3282
3278
  op += oneSeqSize;
3283
3279
  }
@@ -3382,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3382
3378
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3383
3379
  }
3384
3380
 
3381
+ static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
3382
+ {
3383
+
3384
+ const BYTE* ip = (const BYTE*)src;
3385
+ size_t remainingSize = srcSize;
3386
+ U32 magicNumber;
3387
+ blockProperties_t blockProperties;
3388
+
3389
+ /* Frame Header */
3390
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
3391
+ magicNumber = MEM_readLE32(src);
3392
+ if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
3393
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3394
+
3395
+ /* Loop on each block */
3396
+ while (1)
3397
+ {
3398
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3399
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
3400
+
3401
+ ip += ZSTD_blockHeaderSize;
3402
+ remainingSize -= ZSTD_blockHeaderSize;
3403
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3404
+
3405
+ if (cBlockSize == 0) break; /* bt_end */
3406
+
3407
+ ip += cBlockSize;
3408
+ remainingSize -= cBlockSize;
3409
+ }
3410
+
3411
+ return ip - (const BYTE*)src;
3412
+ }
3385
3413
 
3386
3414
  /*******************************
3387
3415
  * Streaming Decompression API
@@ -3496,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3496
3524
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3497
3525
  }
3498
3526
 
3527
+ size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
3528
+ {
3529
+ return ZSTD_findFrameCompressedSize(src, compressedSize);
3530
+ }
3531
+
3499
3532
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3500
3533
  {
3501
3534
  return (ZSTDv02_Dctx*)ZSTD_createDCtx();
@@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
34
34
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
+ /**
38
+ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv02_isError())
42
+ */
43
+ size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
37
45
  /**
38
46
  ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
39
47
  */
@@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2449
2449
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2450
2450
 
2451
2451
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2452
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2452
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2453
2453
  {
2454
2454
  const BYTE* ip = (const BYTE*)src;
2455
2455
  BYTE* op = (BYTE*)dst;
@@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s
2509
2509
  blockType_t bType;
2510
2510
  U32 phase;
2511
2511
  const BYTE* litPtr;
2512
- size_t litBufSize;
2513
2512
  size_t litSize;
2514
2513
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2515
2514
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2581
2580
  size_t litSize = BLOCKSIZE;
2582
2581
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2583
2582
  dctx->litPtr = dctx->litBuffer;
2584
- dctx->litBufSize = BLOCKSIZE;
2585
2583
  dctx->litSize = litSize;
2584
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2586
2585
  return readSize; /* works if it's an error too */
2587
2586
  }
2588
2587
  case IS_RAW:
@@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2593
2592
  if (litSize > srcSize-3) return ERROR(corruption_detected);
2594
2593
  memcpy(dctx->litBuffer, istart, litSize);
2595
2594
  dctx->litPtr = dctx->litBuffer;
2596
- dctx->litBufSize = BLOCKSIZE;
2597
2595
  dctx->litSize = litSize;
2596
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2598
2597
  return litSize+3;
2599
2598
  }
2600
2599
  /* direct reference into compressed stream */
2601
2600
  dctx->litPtr = istart+3;
2602
- dctx->litBufSize = srcSize-3;
2603
2601
  dctx->litSize = litSize;
2604
2602
  return litSize+3;
2605
2603
  }
@@ -2607,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2607
2605
  {
2608
2606
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2609
2607
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2610
- memset(dctx->litBuffer, istart[3], litSize);
2608
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2611
2609
  dctx->litPtr = dctx->litBuffer;
2612
- dctx->litBufSize = BLOCKSIZE;
2613
2610
  dctx->litSize = litSize;
2614
2611
  return 4;
2615
2612
  }
@@ -2816,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2816
2813
  /* checks */
2817
2814
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
2818
2815
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
2819
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2816
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2820
2817
 
2821
2818
  /* copy Literals */
2822
2819
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -2850,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2850
2847
  }
2851
2848
  op += 8; match += 8;
2852
2849
 
2853
- if (oMatchEnd > oend-12)
2850
+ if (oMatchEnd > oend-(16-MINMATCH))
2854
2851
  {
2855
2852
  if (op < oend_8)
2856
2853
  {
@@ -2862,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2862
2859
  }
2863
2860
  else
2864
2861
  {
2865
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
2862
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
2866
2863
  }
2867
2864
  }
2868
2865
 
@@ -2882,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
2882
2879
  BYTE* const oend = ostart + maxDstSize;
2883
2880
  size_t errorCode, dumpsLength;
2884
2881
  const BYTE* litPtr = dctx->litPtr;
2885
- const BYTE* const litMax = litPtr + dctx->litBufSize;
2886
2882
  const BYTE* const litEnd = litPtr + dctx->litSize;
2887
2883
  int nbSeq;
2888
2884
  const BYTE* dumps;
@@ -2918,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
2918
2914
  size_t oneSeqSize;
2919
2915
  nbSeq--;
2920
2916
  ZSTD_decodeSequence(&sequence, &seqState);
2921
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
2917
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
2922
2918
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
2923
2919
  op += oneSeqSize;
2924
2920
  }
@@ -3023,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3023
3019
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3024
3020
  }
3025
3021
 
3022
+ static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
3023
+ {
3024
+ const BYTE* ip = (const BYTE*)src;
3025
+ size_t remainingSize = srcSize;
3026
+ U32 magicNumber;
3027
+ blockProperties_t blockProperties;
3028
+
3029
+ /* Frame Header */
3030
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
3031
+ magicNumber = MEM_readLE32(src);
3032
+ if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
3033
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3034
+
3035
+ /* Loop on each block */
3036
+ while (1)
3037
+ {
3038
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3039
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
3040
+
3041
+ ip += ZSTD_blockHeaderSize;
3042
+ remainingSize -= ZSTD_blockHeaderSize;
3043
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3044
+
3045
+ if (cBlockSize == 0) break; /* bt_end */
3046
+
3047
+ ip += cBlockSize;
3048
+ remainingSize -= cBlockSize;
3049
+ }
3050
+
3051
+ return ip - (const BYTE*)src;
3052
+ }
3053
+
3026
3054
 
3027
3055
  /*******************************
3028
3056
  * Streaming Decompression API
@@ -3137,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
3137
3165
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3138
3166
  }
3139
3167
 
3168
+ size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
3169
+ {
3170
+ return ZSTD_findFrameCompressedSize(src, srcSize);
3171
+ }
3172
+
3140
3173
  ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
3141
3174
  {
3142
3175
  return (ZSTDv03_Dctx*)ZSTD_createDCtx();
@@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
37
  /**
38
+ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv03_isError())
42
+ */
43
+ size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
45
+ /**
38
46
  ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
39
47
  */
40
48
  unsigned ZSTDv03_isError(size_t code);
@@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
487
487
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
488
488
 
489
489
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
490
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
490
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
491
491
  {
492
492
  const BYTE* ip = (const BYTE*)src;
493
493
  BYTE* op = (BYTE*)dst;
@@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s
2706
2706
  blockType_t bType;
2707
2707
  ZSTD_dStage stage;
2708
2708
  const BYTE* litPtr;
2709
- size_t litBufSize;
2710
2709
  size_t litSize;
2711
2710
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2712
2711
  BYTE headerBuffer[ZSTD_frameHeaderSize_max];
@@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
2847
2846
  size_t litSize = BLOCKSIZE;
2848
2847
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2849
2848
  dctx->litPtr = dctx->litBuffer;
2850
- dctx->litBufSize = BLOCKSIZE+8;
2851
2849
  dctx->litSize = litSize;
2850
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2852
2851
  return readSize; /* works if it's an error too */
2853
2852
  }
2854
2853
  case IS_RAW:
@@ -2859,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
2859
2858
  if (litSize > srcSize-3) return ERROR(corruption_detected);
2860
2859
  memcpy(dctx->litBuffer, istart, litSize);
2861
2860
  dctx->litPtr = dctx->litBuffer;
2862
- dctx->litBufSize = BLOCKSIZE+8;
2863
2861
  dctx->litSize = litSize;
2862
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2864
2863
  return litSize+3;
2865
2864
  }
2866
2865
  /* direct reference into compressed stream */
2867
2866
  dctx->litPtr = istart+3;
2868
- dctx->litBufSize = srcSize-3;
2869
2867
  dctx->litSize = litSize;
2870
2868
  return litSize+3; }
2871
2869
  case IS_RLE:
2872
2870
  {
2873
2871
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2874
2872
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2875
- memset(dctx->litBuffer, istart[3], litSize);
2873
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2876
2874
  dctx->litPtr = dctx->litBuffer;
2877
- dctx->litBufSize = BLOCKSIZE+8;
2878
2875
  dctx->litSize = litSize;
2879
2876
  return 4;
2880
2877
  }
@@ -3015,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3015
3012
  /* Literal length */
3016
3013
  litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
3017
3014
  prevOffset = litLength ? seq->offset : seqState->prevOffset;
3018
- if (litLength == MaxLL)
3019
- {
3015
+ if (litLength == MaxLL) {
3020
3016
  U32 add = *dumps++;
3021
3017
  if (add < 255) litLength += add;
3022
- else
3023
- {
3024
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3018
+ else {
3019
+ litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
3025
3020
  dumps += 3;
3026
3021
  }
3027
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3022
+ if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
3023
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3028
3024
  }
3029
3025
 
3030
3026
  /* Offset */
3031
- {
3032
- static const U32 offsetPrefix[MaxOff+1] = {
3027
+ { static const U32 offsetPrefix[MaxOff+1] = {
3033
3028
  1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
3034
3029
  512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
3035
3030
  524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
@@ -3046,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3046
3041
 
3047
3042
  /* MatchLength */
3048
3043
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3049
- if (matchLength == MaxML)
3050
- {
3044
+ if (matchLength == MaxML) {
3051
3045
  U32 add = *dumps++;
3052
3046
  if (add < 255) matchLength += add;
3053
- else
3054
- {
3055
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3047
+ else {
3048
+ matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
3056
3049
  dumps += 3;
3057
3050
  }
3058
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3051
+ if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
3052
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3059
3053
  }
3060
3054
  matchLength += MINMATCH;
3061
3055
 
@@ -3069,7 +3063,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3069
3063
 
3070
3064
  static size_t ZSTD_execSequence(BYTE* op,
3071
3065
  BYTE* const oend, seq_t sequence,
3072
- const BYTE** litPtr, const BYTE* const litLimit_8,
3066
+ const BYTE** litPtr, const BYTE* const litLimit,
3073
3067
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
3074
3068
  {
3075
3069
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@@ -3084,7 +3078,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3084
3078
  /* check */
3085
3079
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3086
3080
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3087
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3081
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3088
3082
 
3089
3083
  /* copy Literals */
3090
3084
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3110,7 +3104,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3110
3104
  op = oLitEnd + length1;
3111
3105
  sequence.matchLength -= length1;
3112
3106
  match = base;
3113
- if (op > oend_8) {
3107
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
3114
3108
  while (op < oMatchEnd) *op++ = *match++;
3115
3109
  return sequenceLength;
3116
3110
  }
@@ -3119,8 +3113,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3119
3113
  /* Requirement: op <= oend_8 */
3120
3114
 
3121
3115
  /* match within prefix */
3122
- if (sequence.offset < 8)
3123
- {
3116
+ if (sequence.offset < 8) {
3124
3117
  /* close range match, overlap */
3125
3118
  const int sub2 = dec64table[sequence.offset];
3126
3119
  op[0] = match[0];
@@ -3130,14 +3123,12 @@ static size_t ZSTD_execSequence(BYTE* op,
3130
3123
  match += dec32table[sequence.offset];
3131
3124
  ZSTD_copy4(op+4, match);
3132
3125
  match -= sub2;
3133
- }
3134
- else
3135
- {
3126
+ } else {
3136
3127
  ZSTD_copy8(op, match);
3137
3128
  }
3138
3129
  op += 8; match += 8;
3139
3130
 
3140
- if (oMatchEnd > oend-12)
3131
+ if (oMatchEnd > oend-(16-MINMATCH))
3141
3132
  {
3142
3133
  if (op < oend_8)
3143
3134
  {
@@ -3149,7 +3140,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3149
3140
  }
3150
3141
  else
3151
3142
  {
3152
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3143
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3153
3144
  }
3154
3145
  return sequenceLength;
3155
3146
  }
@@ -3167,7 +3158,6 @@ static size_t ZSTD_decompressSequences(
3167
3158
  BYTE* const oend = ostart + maxDstSize;
3168
3159
  size_t errorCode, dumpsLength;
3169
3160
  const BYTE* litPtr = dctx->litPtr;
3170
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
3171
3161
  const BYTE* const litEnd = litPtr + dctx->litSize;
3172
3162
  int nbSeq;
3173
3163
  const BYTE* dumps;
@@ -3206,7 +3196,7 @@ static size_t ZSTD_decompressSequences(
3206
3196
  size_t oneSeqSize;
3207
3197
  nbSeq--;
3208
3198
  ZSTD_decodeSequence(&sequence, &seqState);
3209
- oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
3199
+ oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
3210
3200
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3211
3201
  op += oneSeqSize;
3212
3202
  }
@@ -3336,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
3336
3326
  return op-ostart;
3337
3327
  }
3338
3328
 
3329
+ static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
3330
+ {
3331
+ const BYTE* ip = (const BYTE*)src;
3332
+ size_t remainingSize = srcSize;
3333
+ blockProperties_t blockProperties;
3334
+
3335
+ /* Frame Header */
3336
+ if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
3337
+ if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
3338
+ ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
3339
+
3340
+ /* Loop on each block */
3341
+ while (1)
3342
+ {
3343
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3344
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
3345
+
3346
+ ip += ZSTD_blockHeaderSize;
3347
+ remainingSize -= ZSTD_blockHeaderSize;
3348
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3349
+
3350
+ if (cBlockSize == 0) break; /* bt_end */
3351
+
3352
+ ip += cBlockSize;
3353
+ remainingSize -= cBlockSize;
3354
+ }
3355
+
3356
+ return ip - (const BYTE*)src;
3357
+ }
3339
3358
 
3340
3359
  /* ******************************
3341
3360
  * Streaming Decompression API
@@ -3763,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3763
3782
  #endif
3764
3783
  }
3765
3784
 
3785
+ size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
3786
+ {
3787
+ return ZSTD_findFrameCompressedSize(src, srcSize);
3788
+ }
3766
3789
 
3767
3790
  size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
3768
3791
 
@@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
34
34
  size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
+ /**
38
+ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv04_isError())
42
+ */
43
+ size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
37
45
  /**
38
46
  ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
39
47
  */