extzstd 0.1 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja +5 -0
  3. data/README.md +5 -5
  4. data/contrib/zstd/CONTRIBUTING.md +42 -0
  5. data/contrib/zstd/LICENSE-examples +11 -0
  6. data/contrib/zstd/Makefile +315 -0
  7. data/contrib/zstd/NEWS +261 -0
  8. data/contrib/zstd/PATENTS +33 -0
  9. data/contrib/zstd/README.md +121 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +178 -0
  12. data/contrib/zstd/circle.yml +75 -0
  13. data/contrib/zstd/lib/BUCK +186 -0
  14. data/contrib/zstd/lib/Makefile +163 -0
  15. data/contrib/zstd/lib/README.md +77 -0
  16. data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
  17. data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
  18. data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
  19. data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
  20. data/contrib/zstd/{common → lib/common}/fse.h +94 -34
  21. data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
  22. data/contrib/zstd/{common → lib/common}/huf.h +52 -20
  23. data/contrib/zstd/{common → lib/common}/mem.h +17 -13
  24. data/contrib/zstd/lib/common/pool.c +194 -0
  25. data/contrib/zstd/lib/common/pool.h +56 -0
  26. data/contrib/zstd/lib/common/threading.c +80 -0
  27. data/contrib/zstd/lib/common/threading.h +104 -0
  28. data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
  29. data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
  30. data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
  31. data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
  32. data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
  33. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
  34. data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
  35. data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
  36. data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
  37. data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
  38. data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
  39. data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
  40. data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
  41. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
  42. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  43. data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
  44. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
  45. data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
  46. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
  47. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  48. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
  49. data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
  50. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
  51. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
  52. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
  53. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
  54. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
  55. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
  56. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
  57. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
  58. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
  59. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
  60. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
  61. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
  62. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
  63. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
  64. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
  65. data/contrib/zstd/lib/libzstd.pc.in +14 -0
  66. data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
  67. data/ext/depend +2 -0
  68. data/ext/extconf.rb +4 -4
  69. data/ext/extzstd.c +1 -1
  70. data/ext/zstd_common.c +5 -5
  71. data/ext/zstd_compress.c +3 -3
  72. data/ext/zstd_decompress.c +2 -2
  73. data/ext/zstd_dictbuilder.c +2 -2
  74. data/ext/zstd_legacy_v01.c +1 -1
  75. data/ext/zstd_legacy_v02.c +1 -1
  76. data/ext/zstd_legacy_v03.c +1 -1
  77. data/ext/zstd_legacy_v04.c +1 -1
  78. data/ext/zstd_legacy_v05.c +1 -1
  79. data/ext/zstd_legacy_v06.c +1 -1
  80. data/ext/zstd_legacy_v07.c +1 -1
  81. data/gemstub.rb +9 -5
  82. data/lib/extzstd/version.rb +1 -1
  83. metadata +73 -51
  84. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  85. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  86. data/contrib/zstd/dictBuilder/zdict.h +0 -111
@@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
34
34
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
+ /**
38
+ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv01_isError())
42
+ */
43
+ size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
37
45
  /**
38
46
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
39
47
  */
@@ -2808,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2808
2808
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2809
2809
 
2810
2810
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2811
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2811
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2812
2812
  {
2813
2813
  const BYTE* ip = (const BYTE*)src;
2814
2814
  BYTE* op = (BYTE*)dst;
@@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s
2868
2868
  blockType_t bType;
2869
2869
  U32 phase;
2870
2870
  const BYTE* litPtr;
2871
- size_t litBufSize;
2872
2871
  size_t litSize;
2873
2872
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2874
2873
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2940
2939
  size_t litSize = BLOCKSIZE;
2941
2940
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2942
2941
  dctx->litPtr = dctx->litBuffer;
2943
- dctx->litBufSize = BLOCKSIZE;
2944
2942
  dctx->litSize = litSize;
2943
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2945
2944
  return readSize; /* works if it's an error too */
2946
2945
  }
2947
2946
  case IS_RAW:
@@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2952
2951
  if (litSize > srcSize-3) return ERROR(corruption_detected);
2953
2952
  memcpy(dctx->litBuffer, istart, litSize);
2954
2953
  dctx->litPtr = dctx->litBuffer;
2955
- dctx->litBufSize = BLOCKSIZE;
2956
2954
  dctx->litSize = litSize;
2955
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2957
2956
  return litSize+3;
2958
2957
  }
2959
2958
  /* direct reference into compressed stream */
2960
2959
  dctx->litPtr = istart+3;
2961
- dctx->litBufSize = srcSize-3;
2962
2960
  dctx->litSize = litSize;
2963
2961
  return litSize+3;
2964
2962
  }
@@ -2966,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2966
2964
  {
2967
2965
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2968
2966
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2969
- memset(dctx->litBuffer, istart[3], litSize);
2967
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2970
2968
  dctx->litPtr = dctx->litBuffer;
2971
- dctx->litBufSize = BLOCKSIZE;
2972
2969
  dctx->litSize = litSize;
2973
2970
  return 4;
2974
2971
  }
@@ -3175,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3175
3172
  /* checks */
3176
3173
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3177
3174
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3178
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3175
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3179
3176
 
3180
3177
  /* copy Literals */
3181
3178
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3209,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3209
3206
  }
3210
3207
  op += 8; match += 8;
3211
3208
 
3212
- if (oMatchEnd > oend-12)
3209
+ if (oMatchEnd > oend-(16-MINMATCH))
3213
3210
  {
3214
3211
  if (op < oend_8)
3215
3212
  {
@@ -3221,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3221
3218
  }
3222
3219
  else
3223
3220
  {
3224
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3221
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3225
3222
  }
3226
3223
  }
3227
3224
 
@@ -3241,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
3241
3238
  BYTE* const oend = ostart + maxDstSize;
3242
3239
  size_t errorCode, dumpsLength;
3243
3240
  const BYTE* litPtr = dctx->litPtr;
3244
- const BYTE* const litMax = litPtr + dctx->litBufSize;
3245
3241
  const BYTE* const litEnd = litPtr + dctx->litSize;
3246
3242
  int nbSeq;
3247
3243
  const BYTE* dumps;
@@ -3277,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
3277
3273
  size_t oneSeqSize;
3278
3274
  nbSeq--;
3279
3275
  ZSTD_decodeSequence(&sequence, &seqState);
3280
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
3276
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
3281
3277
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3282
3278
  op += oneSeqSize;
3283
3279
  }
@@ -3382,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3382
3378
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3383
3379
  }
3384
3380
 
3381
+ static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
3382
+ {
3383
+
3384
+ const BYTE* ip = (const BYTE*)src;
3385
+ size_t remainingSize = srcSize;
3386
+ U32 magicNumber;
3387
+ blockProperties_t blockProperties;
3388
+
3389
+ /* Frame Header */
3390
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
3391
+ magicNumber = MEM_readLE32(src);
3392
+ if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
3393
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3394
+
3395
+ /* Loop on each block */
3396
+ while (1)
3397
+ {
3398
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3399
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
3400
+
3401
+ ip += ZSTD_blockHeaderSize;
3402
+ remainingSize -= ZSTD_blockHeaderSize;
3403
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3404
+
3405
+ if (cBlockSize == 0) break; /* bt_end */
3406
+
3407
+ ip += cBlockSize;
3408
+ remainingSize -= cBlockSize;
3409
+ }
3410
+
3411
+ return ip - (const BYTE*)src;
3412
+ }
3385
3413
 
3386
3414
  /*******************************
3387
3415
  * Streaming Decompression API
@@ -3496,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3496
3524
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3497
3525
  }
3498
3526
 
3527
+ size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
3528
+ {
3529
+ return ZSTD_findFrameCompressedSize(src, compressedSize);
3530
+ }
3531
+
3499
3532
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3500
3533
  {
3501
3534
  return (ZSTDv02_Dctx*)ZSTD_createDCtx();
@@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
34
34
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
+ /**
38
+ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv02_isError())
42
+ */
43
+ size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
37
45
  /**
38
46
  ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
39
47
  */
@@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2449
2449
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2450
2450
 
2451
2451
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2452
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2452
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2453
2453
  {
2454
2454
  const BYTE* ip = (const BYTE*)src;
2455
2455
  BYTE* op = (BYTE*)dst;
@@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s
2509
2509
  blockType_t bType;
2510
2510
  U32 phase;
2511
2511
  const BYTE* litPtr;
2512
- size_t litBufSize;
2513
2512
  size_t litSize;
2514
2513
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2515
2514
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2581
2580
  size_t litSize = BLOCKSIZE;
2582
2581
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2583
2582
  dctx->litPtr = dctx->litBuffer;
2584
- dctx->litBufSize = BLOCKSIZE;
2585
2583
  dctx->litSize = litSize;
2584
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2586
2585
  return readSize; /* works if it's an error too */
2587
2586
  }
2588
2587
  case IS_RAW:
@@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2593
2592
  if (litSize > srcSize-3) return ERROR(corruption_detected);
2594
2593
  memcpy(dctx->litBuffer, istart, litSize);
2595
2594
  dctx->litPtr = dctx->litBuffer;
2596
- dctx->litBufSize = BLOCKSIZE;
2597
2595
  dctx->litSize = litSize;
2596
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2598
2597
  return litSize+3;
2599
2598
  }
2600
2599
  /* direct reference into compressed stream */
2601
2600
  dctx->litPtr = istart+3;
2602
- dctx->litBufSize = srcSize-3;
2603
2601
  dctx->litSize = litSize;
2604
2602
  return litSize+3;
2605
2603
  }
@@ -2607,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2607
2605
  {
2608
2606
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2609
2607
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2610
- memset(dctx->litBuffer, istart[3], litSize);
2608
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2611
2609
  dctx->litPtr = dctx->litBuffer;
2612
- dctx->litBufSize = BLOCKSIZE;
2613
2610
  dctx->litSize = litSize;
2614
2611
  return 4;
2615
2612
  }
@@ -2816,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2816
2813
  /* checks */
2817
2814
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
2818
2815
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
2819
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2816
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2820
2817
 
2821
2818
  /* copy Literals */
2822
2819
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -2850,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2850
2847
  }
2851
2848
  op += 8; match += 8;
2852
2849
 
2853
- if (oMatchEnd > oend-12)
2850
+ if (oMatchEnd > oend-(16-MINMATCH))
2854
2851
  {
2855
2852
  if (op < oend_8)
2856
2853
  {
@@ -2862,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2862
2859
  }
2863
2860
  else
2864
2861
  {
2865
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
2862
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
2866
2863
  }
2867
2864
  }
2868
2865
 
@@ -2882,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
2882
2879
  BYTE* const oend = ostart + maxDstSize;
2883
2880
  size_t errorCode, dumpsLength;
2884
2881
  const BYTE* litPtr = dctx->litPtr;
2885
- const BYTE* const litMax = litPtr + dctx->litBufSize;
2886
2882
  const BYTE* const litEnd = litPtr + dctx->litSize;
2887
2883
  int nbSeq;
2888
2884
  const BYTE* dumps;
@@ -2918,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
2918
2914
  size_t oneSeqSize;
2919
2915
  nbSeq--;
2920
2916
  ZSTD_decodeSequence(&sequence, &seqState);
2921
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
2917
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
2922
2918
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
2923
2919
  op += oneSeqSize;
2924
2920
  }
@@ -3023,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3023
3019
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3024
3020
  }
3025
3021
 
3022
+ static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
3023
+ {
3024
+ const BYTE* ip = (const BYTE*)src;
3025
+ size_t remainingSize = srcSize;
3026
+ U32 magicNumber;
3027
+ blockProperties_t blockProperties;
3028
+
3029
+ /* Frame Header */
3030
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
3031
+ magicNumber = MEM_readLE32(src);
3032
+ if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
3033
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3034
+
3035
+ /* Loop on each block */
3036
+ while (1)
3037
+ {
3038
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3039
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
3040
+
3041
+ ip += ZSTD_blockHeaderSize;
3042
+ remainingSize -= ZSTD_blockHeaderSize;
3043
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3044
+
3045
+ if (cBlockSize == 0) break; /* bt_end */
3046
+
3047
+ ip += cBlockSize;
3048
+ remainingSize -= cBlockSize;
3049
+ }
3050
+
3051
+ return ip - (const BYTE*)src;
3052
+ }
3053
+
3026
3054
 
3027
3055
  /*******************************
3028
3056
  * Streaming Decompression API
@@ -3137,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
3137
3165
  return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3138
3166
  }
3139
3167
 
3168
+ size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
3169
+ {
3170
+ return ZSTD_findFrameCompressedSize(src, srcSize);
3171
+ }
3172
+
3140
3173
  ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
3141
3174
  {
3142
3175
  return (ZSTDv03_Dctx*)ZSTD_createDCtx();
@@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
37
  /**
38
+ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv03_isError())
42
+ */
43
+ size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
45
+ /**
38
46
  ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
39
47
  */
40
48
  unsigned ZSTDv03_isError(size_t code);
@@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
487
487
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
488
488
 
489
489
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
490
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
490
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
491
491
  {
492
492
  const BYTE* ip = (const BYTE*)src;
493
493
  BYTE* op = (BYTE*)dst;
@@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s
2706
2706
  blockType_t bType;
2707
2707
  ZSTD_dStage stage;
2708
2708
  const BYTE* litPtr;
2709
- size_t litBufSize;
2710
2709
  size_t litSize;
2711
2710
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2712
2711
  BYTE headerBuffer[ZSTD_frameHeaderSize_max];
@@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
2847
2846
  size_t litSize = BLOCKSIZE;
2848
2847
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2849
2848
  dctx->litPtr = dctx->litBuffer;
2850
- dctx->litBufSize = BLOCKSIZE+8;
2851
2849
  dctx->litSize = litSize;
2850
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2852
2851
  return readSize; /* works if it's an error too */
2853
2852
  }
2854
2853
  case IS_RAW:
@@ -2859,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
2859
2858
  if (litSize > srcSize-3) return ERROR(corruption_detected);
2860
2859
  memcpy(dctx->litBuffer, istart, litSize);
2861
2860
  dctx->litPtr = dctx->litBuffer;
2862
- dctx->litBufSize = BLOCKSIZE+8;
2863
2861
  dctx->litSize = litSize;
2862
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2864
2863
  return litSize+3;
2865
2864
  }
2866
2865
  /* direct reference into compressed stream */
2867
2866
  dctx->litPtr = istart+3;
2868
- dctx->litBufSize = srcSize-3;
2869
2867
  dctx->litSize = litSize;
2870
2868
  return litSize+3; }
2871
2869
  case IS_RLE:
2872
2870
  {
2873
2871
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2874
2872
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2875
- memset(dctx->litBuffer, istart[3], litSize);
2873
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2876
2874
  dctx->litPtr = dctx->litBuffer;
2877
- dctx->litBufSize = BLOCKSIZE+8;
2878
2875
  dctx->litSize = litSize;
2879
2876
  return 4;
2880
2877
  }
@@ -3015,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3015
3012
  /* Literal length */
3016
3013
  litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
3017
3014
  prevOffset = litLength ? seq->offset : seqState->prevOffset;
3018
- if (litLength == MaxLL)
3019
- {
3015
+ if (litLength == MaxLL) {
3020
3016
  U32 add = *dumps++;
3021
3017
  if (add < 255) litLength += add;
3022
- else
3023
- {
3024
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3018
+ else {
3019
+ litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
3025
3020
  dumps += 3;
3026
3021
  }
3027
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3022
+ if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
3023
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3028
3024
  }
3029
3025
 
3030
3026
  /* Offset */
3031
- {
3032
- static const U32 offsetPrefix[MaxOff+1] = {
3027
+ { static const U32 offsetPrefix[MaxOff+1] = {
3033
3028
  1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
3034
3029
  512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
3035
3030
  524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
@@ -3046,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3046
3041
 
3047
3042
  /* MatchLength */
3048
3043
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3049
- if (matchLength == MaxML)
3050
- {
3044
+ if (matchLength == MaxML) {
3051
3045
  U32 add = *dumps++;
3052
3046
  if (add < 255) matchLength += add;
3053
- else
3054
- {
3055
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3047
+ else {
3048
+ matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
3056
3049
  dumps += 3;
3057
3050
  }
3058
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3051
+ if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
3052
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3059
3053
  }
3060
3054
  matchLength += MINMATCH;
3061
3055
 
@@ -3069,7 +3063,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3069
3063
 
3070
3064
  static size_t ZSTD_execSequence(BYTE* op,
3071
3065
  BYTE* const oend, seq_t sequence,
3072
- const BYTE** litPtr, const BYTE* const litLimit_8,
3066
+ const BYTE** litPtr, const BYTE* const litLimit,
3073
3067
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
3074
3068
  {
3075
3069
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@@ -3084,7 +3078,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3084
3078
  /* check */
3085
3079
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3086
3080
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3087
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3081
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3088
3082
 
3089
3083
  /* copy Literals */
3090
3084
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3110,7 +3104,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3110
3104
  op = oLitEnd + length1;
3111
3105
  sequence.matchLength -= length1;
3112
3106
  match = base;
3113
- if (op > oend_8) {
3107
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
3114
3108
  while (op < oMatchEnd) *op++ = *match++;
3115
3109
  return sequenceLength;
3116
3110
  }
@@ -3119,8 +3113,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3119
3113
  /* Requirement: op <= oend_8 */
3120
3114
 
3121
3115
  /* match within prefix */
3122
- if (sequence.offset < 8)
3123
- {
3116
+ if (sequence.offset < 8) {
3124
3117
  /* close range match, overlap */
3125
3118
  const int sub2 = dec64table[sequence.offset];
3126
3119
  op[0] = match[0];
@@ -3130,14 +3123,12 @@ static size_t ZSTD_execSequence(BYTE* op,
3130
3123
  match += dec32table[sequence.offset];
3131
3124
  ZSTD_copy4(op+4, match);
3132
3125
  match -= sub2;
3133
- }
3134
- else
3135
- {
3126
+ } else {
3136
3127
  ZSTD_copy8(op, match);
3137
3128
  }
3138
3129
  op += 8; match += 8;
3139
3130
 
3140
- if (oMatchEnd > oend-12)
3131
+ if (oMatchEnd > oend-(16-MINMATCH))
3141
3132
  {
3142
3133
  if (op < oend_8)
3143
3134
  {
@@ -3149,7 +3140,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3149
3140
  }
3150
3141
  else
3151
3142
  {
3152
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3143
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3153
3144
  }
3154
3145
  return sequenceLength;
3155
3146
  }
@@ -3167,7 +3158,6 @@ static size_t ZSTD_decompressSequences(
3167
3158
  BYTE* const oend = ostart + maxDstSize;
3168
3159
  size_t errorCode, dumpsLength;
3169
3160
  const BYTE* litPtr = dctx->litPtr;
3170
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
3171
3161
  const BYTE* const litEnd = litPtr + dctx->litSize;
3172
3162
  int nbSeq;
3173
3163
  const BYTE* dumps;
@@ -3206,7 +3196,7 @@ static size_t ZSTD_decompressSequences(
3206
3196
  size_t oneSeqSize;
3207
3197
  nbSeq--;
3208
3198
  ZSTD_decodeSequence(&sequence, &seqState);
3209
- oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
3199
+ oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
3210
3200
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3211
3201
  op += oneSeqSize;
3212
3202
  }
@@ -3336,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
3336
3326
  return op-ostart;
3337
3327
  }
3338
3328
 
3329
+ static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
3330
+ {
3331
+ const BYTE* ip = (const BYTE*)src;
3332
+ size_t remainingSize = srcSize;
3333
+ blockProperties_t blockProperties;
3334
+
3335
+ /* Frame Header */
3336
+ if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
3337
+ if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
3338
+ ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
3339
+
3340
+ /* Loop on each block */
3341
+ while (1)
3342
+ {
3343
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3344
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
3345
+
3346
+ ip += ZSTD_blockHeaderSize;
3347
+ remainingSize -= ZSTD_blockHeaderSize;
3348
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3349
+
3350
+ if (cBlockSize == 0) break; /* bt_end */
3351
+
3352
+ ip += cBlockSize;
3353
+ remainingSize -= cBlockSize;
3354
+ }
3355
+
3356
+ return ip - (const BYTE*)src;
3357
+ }
3339
3358
 
3340
3359
  /* ******************************
3341
3360
  * Streaming Decompression API
@@ -3763,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3763
3782
  #endif
3764
3783
  }
3765
3784
 
3785
+ size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
3786
+ {
3787
+ return ZSTD_findFrameCompressedSize(src, srcSize);
3788
+ }
3766
3789
 
3767
3790
  size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
3768
3791
 
@@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
34
34
  size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
35
35
  const void* src, size_t compressedSize);
36
36
 
37
+ /**
38
+ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
39
+ compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
40
+ return : the number of bytes that would be read to decompress this frame
41
+ or an errorCode if it fails (which can be tested using ZSTDv04_isError())
42
+ */
43
+ size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
44
+
37
45
  /**
38
46
  ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
39
47
  */