extzstd 0.3 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +8 -0
  3. data/README.md +1 -1
  4. data/contrib/zstd/CHANGELOG +94 -0
  5. data/contrib/zstd/CONTRIBUTING.md +351 -1
  6. data/contrib/zstd/Makefile +32 -10
  7. data/contrib/zstd/README.md +33 -10
  8. data/contrib/zstd/TESTING.md +2 -2
  9. data/contrib/zstd/appveyor.yml +42 -4
  10. data/contrib/zstd/lib/Makefile +128 -60
  11. data/contrib/zstd/lib/README.md +47 -16
  12. data/contrib/zstd/lib/common/bitstream.h +38 -39
  13. data/contrib/zstd/lib/common/compiler.h +40 -5
  14. data/contrib/zstd/lib/common/cpu.h +1 -1
  15. data/contrib/zstd/lib/common/debug.c +11 -31
  16. data/contrib/zstd/lib/common/debug.h +11 -31
  17. data/contrib/zstd/lib/common/entropy_common.c +13 -33
  18. data/contrib/zstd/lib/common/error_private.c +2 -1
  19. data/contrib/zstd/lib/common/error_private.h +6 -2
  20. data/contrib/zstd/lib/common/fse.h +12 -32
  21. data/contrib/zstd/lib/common/fse_decompress.c +12 -35
  22. data/contrib/zstd/lib/common/huf.h +15 -33
  23. data/contrib/zstd/lib/common/mem.h +75 -2
  24. data/contrib/zstd/lib/common/pool.c +8 -4
  25. data/contrib/zstd/lib/common/pool.h +2 -2
  26. data/contrib/zstd/lib/common/threading.c +50 -4
  27. data/contrib/zstd/lib/common/threading.h +36 -4
  28. data/contrib/zstd/lib/common/xxhash.c +23 -35
  29. data/contrib/zstd/lib/common/xxhash.h +11 -31
  30. data/contrib/zstd/lib/common/zstd_common.c +1 -1
  31. data/contrib/zstd/lib/common/zstd_errors.h +2 -1
  32. data/contrib/zstd/lib/common/zstd_internal.h +154 -26
  33. data/contrib/zstd/lib/compress/fse_compress.c +17 -40
  34. data/contrib/zstd/lib/compress/hist.c +15 -35
  35. data/contrib/zstd/lib/compress/hist.h +12 -32
  36. data/contrib/zstd/lib/compress/huf_compress.c +92 -92
  37. data/contrib/zstd/lib/compress/zstd_compress.c +1191 -1330
  38. data/contrib/zstd/lib/compress/zstd_compress_internal.h +317 -55
  39. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  40. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  41. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  45. data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
  46. data/contrib/zstd/lib/compress/zstd_double_fast.c +65 -43
  47. data/contrib/zstd/lib/compress/zstd_double_fast.h +2 -2
  48. data/contrib/zstd/lib/compress/zstd_fast.c +92 -66
  49. data/contrib/zstd/lib/compress/zstd_fast.h +2 -2
  50. data/contrib/zstd/lib/compress/zstd_lazy.c +74 -42
  51. data/contrib/zstd/lib/compress/zstd_lazy.h +1 -1
  52. data/contrib/zstd/lib/compress/zstd_ldm.c +32 -10
  53. data/contrib/zstd/lib/compress/zstd_ldm.h +7 -2
  54. data/contrib/zstd/lib/compress/zstd_opt.c +81 -114
  55. data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
  56. data/contrib/zstd/lib/compress/zstdmt_compress.c +95 -51
  57. data/contrib/zstd/lib/compress/zstdmt_compress.h +3 -2
  58. data/contrib/zstd/lib/decompress/huf_decompress.c +76 -60
  59. data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -8
  60. data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
  61. data/contrib/zstd/lib/decompress/zstd_decompress.c +292 -172
  62. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +459 -338
  63. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +3 -3
  64. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +18 -4
  65. data/contrib/zstd/lib/deprecated/zbuff.h +9 -8
  66. data/contrib/zstd/lib/deprecated/zbuff_common.c +2 -2
  67. data/contrib/zstd/lib/deprecated/zbuff_compress.c +1 -1
  68. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +1 -1
  69. data/contrib/zstd/lib/dictBuilder/cover.c +164 -54
  70. data/contrib/zstd/lib/dictBuilder/cover.h +52 -7
  71. data/contrib/zstd/lib/dictBuilder/fastcover.c +60 -43
  72. data/contrib/zstd/lib/dictBuilder/zdict.c +43 -19
  73. data/contrib/zstd/lib/dictBuilder/zdict.h +56 -28
  74. data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -4
  75. data/contrib/zstd/lib/legacy/zstd_v01.c +110 -110
  76. data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
  77. data/contrib/zstd/lib/legacy/zstd_v02.c +23 -13
  78. data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
  79. data/contrib/zstd/lib/legacy/zstd_v03.c +23 -13
  80. data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
  81. data/contrib/zstd/lib/legacy/zstd_v04.c +30 -17
  82. data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
  83. data/contrib/zstd/lib/legacy/zstd_v05.c +113 -102
  84. data/contrib/zstd/lib/legacy/zstd_v05.h +2 -2
  85. data/contrib/zstd/lib/legacy/zstd_v06.c +20 -18
  86. data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
  87. data/contrib/zstd/lib/legacy/zstd_v07.c +25 -19
  88. data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
  89. data/contrib/zstd/lib/libzstd.pc.in +3 -2
  90. data/contrib/zstd/lib/zstd.h +265 -88
  91. data/ext/extzstd.h +1 -1
  92. data/ext/libzstd_conf.h +8 -0
  93. data/ext/zstd_common.c +1 -3
  94. data/ext/zstd_compress.c +3 -3
  95. data/ext/zstd_decompress.c +1 -5
  96. data/ext/zstd_dictbuilder.c +2 -3
  97. data/ext/zstd_dictbuilder_fastcover.c +1 -3
  98. data/ext/zstd_legacy_v01.c +2 -0
  99. data/ext/zstd_legacy_v02.c +2 -0
  100. data/ext/zstd_legacy_v03.c +2 -0
  101. data/ext/zstd_legacy_v04.c +2 -0
  102. data/ext/zstd_legacy_v05.c +2 -0
  103. data/ext/zstd_legacy_v06.c +2 -0
  104. data/ext/zstd_legacy_v07.c +2 -0
  105. data/lib/extzstd.rb +18 -10
  106. data/lib/extzstd/version.rb +1 -1
  107. metadata +15 -6
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,17 +15,17 @@
15
15
  * Dependencies
16
16
  *********************************************************/
17
17
  #include <string.h> /* memcpy, memmove, memset */
18
- #include "cpu.h" /* bmi2 */
19
- #include "mem.h" /* low level memory routines */
18
+ #include "../common/cpu.h" /* bmi2 */
19
+ #include "../common/mem.h" /* low level memory routines */
20
20
  #define FSE_STATIC_LINKING_ONLY
21
- #include "fse.h"
21
+ #include "../common/fse.h"
22
22
  #define HUF_STATIC_LINKING_ONLY
23
- #include "huf.h"
23
+ #include "../common/huf.h"
24
24
  #include "zstd_decompress_internal.h"
25
25
  #include "zstd_ddict.h"
26
26
 
27
27
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
28
- # include "zstd_legacy.h"
28
+ # include "../legacy/zstd_legacy.h"
29
29
  #endif
30
30
 
31
31
 
@@ -65,6 +65,10 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
65
65
  dctx->virtualStart = ddict->dictContent;
66
66
  dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
67
67
  dctx->previousDstEnd = dctx->dictEnd;
68
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
69
+ dctx->dictContentBeginForFuzzing = dctx->prefixStart;
70
+ dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
71
+ #endif
68
72
  if (ddict->entropyPresent) {
69
73
  dctx->litEntropy = 1;
70
74
  dctx->fseEntropy = 1;
@@ -107,7 +111,7 @@ ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
107
111
  /* load entropy tables */
108
112
  RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
109
113
  &ddict->entropy, ddict->dictContent, ddict->dictSize)),
110
- dictionary_corrupted);
114
+ dictionary_corrupted, "");
111
115
  ddict->entropyPresent = 1;
112
116
  return 0;
113
117
  }
@@ -133,7 +137,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
133
137
  ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
134
138
 
135
139
  /* parse dictionary content */
136
- FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
140
+ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
137
141
 
138
142
  return 0;
139
143
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +16,7 @@
16
16
  * Dependencies
17
17
  *********************************************************/
18
18
  #include <stddef.h> /* size_t */
19
- #include "zstd.h" /* ZSTD_DDict, and several public functions */
19
+ #include "../zstd.h" /* ZSTD_DDict, and several public functions */
20
20
 
21
21
 
22
22
  /*-*******************************************************
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -56,19 +56,19 @@
56
56
  * Dependencies
57
57
  *********************************************************/
58
58
  #include <string.h> /* memcpy, memmove, memset */
59
- #include "cpu.h" /* bmi2 */
60
- #include "mem.h" /* low level memory routines */
59
+ #include "../common/cpu.h" /* bmi2 */
60
+ #include "../common/mem.h" /* low level memory routines */
61
61
  #define FSE_STATIC_LINKING_ONLY
62
- #include "fse.h"
62
+ #include "../common/fse.h"
63
63
  #define HUF_STATIC_LINKING_ONLY
64
- #include "huf.h"
65
- #include "zstd_internal.h" /* blockProperties_t */
64
+ #include "../common/huf.h"
65
+ #include "../common/zstd_internal.h" /* blockProperties_t */
66
66
  #include "zstd_decompress_internal.h" /* ZSTD_DCtx */
67
67
  #include "zstd_ddict.h" /* ZSTD_DDictDictContent */
68
68
  #include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
69
69
 
70
70
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
71
- # include "zstd_legacy.h"
71
+ # include "../legacy/zstd_legacy.h"
72
72
  #endif
73
73
 
74
74
 
@@ -88,10 +88,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
88
88
 
89
89
  static size_t ZSTD_startingInputLength(ZSTD_format_e format)
90
90
  {
91
- size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
92
- ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE :
93
- ZSTD_FRAMEHEADERSIZE_PREFIX;
94
- ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
91
+ size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
95
92
  /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
96
93
  assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
97
94
  return startingInputLength;
@@ -114,7 +111,12 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
114
111
  dctx->legacyContext = NULL;
115
112
  dctx->previousLegacyVersion = 0;
116
113
  dctx->noForwardProgress = 0;
114
+ dctx->oversizedDuration = 0;
117
115
  dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
116
+ dctx->outBufferMode = ZSTD_obm_buffered;
117
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
118
+ dctx->dictContentEndForFuzzing = NULL;
119
+ #endif
118
120
  }
119
121
 
120
122
  ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
@@ -211,7 +213,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
211
213
  static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
212
214
  {
213
215
  size_t const minInputSize = ZSTD_startingInputLength(format);
214
- RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong);
216
+ RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
215
217
 
216
218
  { BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
217
219
  U32 const dictID= fhd & 3;
@@ -259,7 +261,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
259
261
  zfhPtr->frameType = ZSTD_skippableFrame;
260
262
  return 0;
261
263
  }
262
- RETURN_ERROR(prefix_unknown);
264
+ RETURN_ERROR(prefix_unknown, "");
263
265
  }
264
266
 
265
267
  /* ensure there is enough `srcSize` to fully read/decode frame header */
@@ -283,7 +285,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
283
285
  if (!singleSegment) {
284
286
  BYTE const wlByte = ip[pos++];
285
287
  U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
286
- RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge);
288
+ RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
287
289
  windowSize = (1ULL << windowLog);
288
290
  windowSize += (windowSize >> 3) * (wlByte&7);
289
291
  }
@@ -355,13 +357,16 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
355
357
  size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
356
358
  U32 sizeU32;
357
359
 
358
- RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong);
360
+ RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
359
361
 
360
362
  sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
361
363
  RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
362
- frameParameter_unsupported);
363
-
364
- return skippableHeaderSize + sizeU32;
364
+ frameParameter_unsupported, "");
365
+ {
366
+ size_t const skippableSize = skippableHeaderSize + sizeU32;
367
+ RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
368
+ return skippableSize;
369
+ }
365
370
  }
366
371
 
367
372
  /** ZSTD_findDecompressedSize() :
@@ -373,16 +378,15 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
373
378
  {
374
379
  unsigned long long totalDstSize = 0;
375
380
 
376
- while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
381
+ while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
377
382
  U32 const magicNumber = MEM_readLE32(src);
378
383
 
379
384
  if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
380
385
  size_t const skippableSize = readSkippableFrameSize(src, srcSize);
381
- if (ZSTD_isError(skippableSize))
382
- return skippableSize;
383
- if (srcSize < skippableSize) {
386
+ if (ZSTD_isError(skippableSize)) {
384
387
  return ZSTD_CONTENTSIZE_ERROR;
385
388
  }
389
+ assert(skippableSize <= srcSize);
386
390
 
387
391
  src = (const BYTE *)src + skippableSize;
388
392
  srcSize -= skippableSize;
@@ -440,7 +444,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
440
444
  * harder.
441
445
  */
442
446
  RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
443
- dictionary_wrong);
447
+ dictionary_wrong, "");
444
448
  #endif
445
449
  if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
446
450
  return 0;
@@ -467,6 +471,8 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
467
471
  if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
468
472
  && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
469
473
  frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
474
+ assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
475
+ frameSizeInfo.compressedSize <= srcSize);
470
476
  return frameSizeInfo;
471
477
  } else {
472
478
  const BYTE* ip = (const BYTE*)src;
@@ -529,7 +535,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
529
535
  return frameSizeInfo.compressedSize;
530
536
  }
531
537
 
532
-
533
538
  /** ZSTD_decompressBound() :
534
539
  * compatible with legacy mode
535
540
  * `src` must point to the start of a ZSTD frame or a skippeable frame
@@ -546,6 +551,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
546
551
  unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
547
552
  if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
548
553
  return ZSTD_CONTENTSIZE_ERROR;
554
+ assert(srcSize >= compressedSize);
549
555
  src = (const BYTE*)src + compressedSize;
550
556
  srcSize -= compressedSize;
551
557
  bound += decompressedBound;
@@ -558,21 +564,11 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
558
564
  * Frame decoding
559
565
  ***************************************************************/
560
566
 
561
-
562
- void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
563
- {
564
- if (dst != dctx->previousDstEnd) { /* not contiguous */
565
- dctx->dictEnd = dctx->previousDstEnd;
566
- dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
567
- dctx->prefixStart = dst;
568
- dctx->previousDstEnd = dst;
569
- }
570
- }
571
-
572
567
  /** ZSTD_insertBlock() :
573
- insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
568
+ * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
574
569
  size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
575
570
  {
571
+ DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
576
572
  ZSTD_checkContinuity(dctx, blockStart);
577
573
  dctx->previousDstEnd = (const char*)blockStart + blockSize;
578
574
  return blockSize;
@@ -585,9 +581,9 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
585
581
  DEBUGLOG(5, "ZSTD_copyRawBlock");
586
582
  if (dst == NULL) {
587
583
  if (srcSize == 0) return 0;
588
- RETURN_ERROR(dstBuffer_null);
584
+ RETURN_ERROR(dstBuffer_null, "");
589
585
  }
590
- RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall);
586
+ RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
591
587
  memcpy(dst, src, srcSize);
592
588
  return srcSize;
593
589
  }
@@ -598,9 +594,9 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
598
594
  {
599
595
  if (dst == NULL) {
600
596
  if (regenSize == 0) return 0;
601
- RETURN_ERROR(dstBuffer_null);
597
+ RETURN_ERROR(dstBuffer_null, "");
602
598
  }
603
- RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall);
599
+ RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
604
600
  memset(dst, b, regenSize);
605
601
  return regenSize;
606
602
  }
@@ -616,7 +612,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
616
612
  {
617
613
  const BYTE* ip = (const BYTE*)(*srcPtr);
618
614
  BYTE* const ostart = (BYTE* const)dst;
619
- BYTE* const oend = ostart + dstCapacity;
615
+ BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
620
616
  BYTE* op = ostart;
621
617
  size_t remainingSrcSize = *srcSizePtr;
622
618
 
@@ -624,15 +620,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
624
620
 
625
621
  /* check */
626
622
  RETURN_ERROR_IF(
627
- remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize,
628
- srcSize_wrong);
623
+ remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
624
+ srcSize_wrong, "");
629
625
 
630
626
  /* Frame Header */
631
- { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX);
627
+ { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
628
+ ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
632
629
  if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
633
630
  RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
634
- srcSize_wrong);
635
- FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
631
+ srcSize_wrong, "");
632
+ FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
636
633
  ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
637
634
  }
638
635
 
@@ -645,7 +642,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
645
642
 
646
643
  ip += ZSTD_blockHeaderSize;
647
644
  remainingSrcSize -= ZSTD_blockHeaderSize;
648
- RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong);
645
+ RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
649
646
 
650
647
  switch(blockProperties.blockType)
651
648
  {
@@ -660,13 +657,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
660
657
  break;
661
658
  case bt_reserved :
662
659
  default:
663
- RETURN_ERROR(corruption_detected);
660
+ RETURN_ERROR(corruption_detected, "invalid block type");
664
661
  }
665
662
 
666
663
  if (ZSTD_isError(decodedSize)) return decodedSize;
667
664
  if (dctx->fParams.checksumFlag)
668
665
  XXH64_update(&dctx->xxhState, op, decodedSize);
669
- op += decodedSize;
666
+ if (decodedSize != 0)
667
+ op += decodedSize;
668
+ assert(ip != NULL);
670
669
  ip += cBlockSize;
671
670
  remainingSrcSize -= cBlockSize;
672
671
  if (blockProperties.lastBlock) break;
@@ -674,14 +673,14 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
674
673
 
675
674
  if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
676
675
  RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
677
- corruption_detected);
676
+ corruption_detected, "");
678
677
  }
679
678
  if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
680
679
  U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
681
680
  U32 checkRead;
682
- RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong);
681
+ RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
683
682
  checkRead = MEM_readLE32(ip);
684
- RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong);
683
+ RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
685
684
  ip += 4;
686
685
  remainingSrcSize -= 4;
687
686
  }
@@ -709,7 +708,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
709
708
  dictSize = ZSTD_DDict_dictSize(ddict);
710
709
  }
711
710
 
712
- while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
711
+ while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
713
712
 
714
713
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
715
714
  if (ZSTD_isLegacy(src, srcSize)) {
@@ -738,9 +737,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
738
737
  (unsigned)magicNumber, ZSTD_MAGICNUMBER);
739
738
  if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
740
739
  size_t const skippableSize = readSkippableFrameSize(src, srcSize);
741
- if (ZSTD_isError(skippableSize))
742
- return skippableSize;
743
- RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong);
740
+ FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
741
+ assert(skippableSize <= srcSize);
744
742
 
745
743
  src = (const BYTE *)src + skippableSize;
746
744
  srcSize -= skippableSize;
@@ -749,11 +747,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
749
747
 
750
748
  if (ddict) {
751
749
  /* we were called from ZSTD_decompress_usingDDict */
752
- FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict));
750
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
753
751
  } else {
754
752
  /* this will initialize correctly with no dict if dict == NULL, so
755
753
  * use this in all cases but ddict */
756
- FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
754
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
757
755
  }
758
756
  ZSTD_checkContinuity(dctx, dst);
759
757
 
@@ -774,7 +772,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
774
772
  "error.");
775
773
  if (ZSTD_isError(res)) return res;
776
774
  assert(res <= dstCapacity);
777
- dst = (BYTE*)dst + res;
775
+ if (res != 0)
776
+ dst = (BYTE*)dst + res;
778
777
  dstCapacity -= res;
779
778
  }
780
779
  moreThan1Frame = 1;
@@ -822,7 +821,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
822
821
  #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
823
822
  size_t regenSize;
824
823
  ZSTD_DCtx* const dctx = ZSTD_createDCtx();
825
- RETURN_ERROR_IF(dctx==NULL, memory_allocation);
824
+ RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
826
825
  regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
827
826
  ZSTD_freeDCtx(dctx);
828
827
  return regenSize;
@@ -840,6 +839,24 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
840
839
  ****************************************/
841
840
  size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
842
841
 
842
+ /**
843
+ * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed,
844
+ * we allow taking a partial block as the input. Currently only raw uncompressed blocks can
845
+ * be streamed.
846
+ *
847
+ * For blocks that can be streamed, this allows us to reduce the latency until we produce
848
+ * output, and avoid copying the input.
849
+ *
850
+ * @param inputSize - The total amount of input that the caller currently has.
851
+ */
852
+ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
853
+ if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))
854
+ return dctx->expected;
855
+ if (dctx->bType != bt_raw)
856
+ return dctx->expected;
857
+ return MIN(MAX(inputSize, 1), dctx->expected);
858
+ }
859
+
843
860
  ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
844
861
  switch(dctx->stage)
845
862
  {
@@ -872,7 +889,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
872
889
  {
873
890
  DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
874
891
  /* Sanity check */
875
- RETURN_ERROR_IF(srcSize != dctx->expected, srcSize_wrong, "not allowed");
892
+ RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
876
893
  if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
877
894
 
878
895
  switch (dctx->stage)
@@ -897,7 +914,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
897
914
  case ZSTDds_decodeFrameHeader:
898
915
  assert(src != NULL);
899
916
  memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
900
- FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
917
+ FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
901
918
  dctx->expected = ZSTD_blockHeaderSize;
902
919
  dctx->stage = ZSTDds_decodeBlockHeader;
903
920
  return 0;
@@ -906,6 +923,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
906
923
  { blockProperties_t bp;
907
924
  size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
908
925
  if (ZSTD_isError(cBlockSize)) return cBlockSize;
926
+ RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum");
909
927
  dctx->expected = cBlockSize;
910
928
  dctx->bType = bp.blockType;
911
929
  dctx->rleSize = bp.origSize;
@@ -938,28 +956,41 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
938
956
  case bt_compressed:
939
957
  DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
940
958
  rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1);
959
+ dctx->expected = 0; /* Streaming not supported */
941
960
  break;
942
961
  case bt_raw :
962
+ assert(srcSize <= dctx->expected);
943
963
  rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
964
+ FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
965
+ assert(rSize == srcSize);
966
+ dctx->expected -= rSize;
944
967
  break;
945
968
  case bt_rle :
946
969
  rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize);
970
+ dctx->expected = 0; /* Streaming not supported */
947
971
  break;
948
972
  case bt_reserved : /* should never happen */
949
973
  default:
950
- RETURN_ERROR(corruption_detected);
974
+ RETURN_ERROR(corruption_detected, "invalid block type");
951
975
  }
952
- if (ZSTD_isError(rSize)) return rSize;
976
+ FORWARD_IF_ERROR(rSize, "");
977
+ RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
953
978
  DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
954
979
  dctx->decodedSize += rSize;
955
980
  if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
981
+ dctx->previousDstEnd = (char*)dst + rSize;
982
+
983
+ /* Stay on the same stage until we are finished streaming the block. */
984
+ if (dctx->expected > 0) {
985
+ return rSize;
986
+ }
956
987
 
957
988
  if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
958
989
  DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
959
990
  RETURN_ERROR_IF(
960
991
  dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
961
992
  && dctx->decodedSize != dctx->fParams.frameContentSize,
962
- corruption_detected);
993
+ corruption_detected, "");
963
994
  if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
964
995
  dctx->expected = 4;
965
996
  dctx->stage = ZSTDds_checkChecksum;
@@ -970,7 +1001,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
970
1001
  } else {
971
1002
  dctx->stage = ZSTDds_decodeBlockHeader;
972
1003
  dctx->expected = ZSTD_blockHeaderSize;
973
- dctx->previousDstEnd = (char*)dst + rSize;
974
1004
  }
975
1005
  return rSize;
976
1006
  }
@@ -980,7 +1010,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
980
1010
  { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
981
1011
  U32 const check32 = MEM_readLE32(src);
982
1012
  DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
983
- RETURN_ERROR_IF(check32 != h32, checksum_wrong);
1013
+ RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
984
1014
  dctx->expected = 0;
985
1015
  dctx->stage = ZSTDds_getFrameHeaderSize;
986
1016
  return 0;
@@ -1001,7 +1031,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
1001
1031
 
1002
1032
  default:
1003
1033
  assert(0); /* impossible */
1004
- RETURN_ERROR(GENERIC); /* some compiler require default to do something */
1034
+ RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
1005
1035
  }
1006
1036
  }
1007
1037
 
@@ -1012,6 +1042,10 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
1012
1042
  dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
1013
1043
  dctx->prefixStart = dict;
1014
1044
  dctx->previousDstEnd = (const char*)dict + dictSize;
1045
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1046
+ dctx->dictContentBeginForFuzzing = dctx->prefixStart;
1047
+ dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
1048
+ #endif
1015
1049
  return 0;
1016
1050
  }
1017
1051
 
@@ -1025,7 +1059,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1025
1059
  const BYTE* dictPtr = (const BYTE*)dict;
1026
1060
  const BYTE* const dictEnd = dictPtr + dictSize;
1027
1061
 
1028
- RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted);
1062
+ RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
1029
1063
  assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
1030
1064
  dictPtr += 8; /* skip header = magic + dictID */
1031
1065
 
@@ -1044,16 +1078,16 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1044
1078
  dictPtr, dictEnd - dictPtr,
1045
1079
  workspace, workspaceSize);
1046
1080
  #endif
1047
- RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted);
1081
+ RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
1048
1082
  dictPtr += hSize;
1049
1083
  }
1050
1084
 
1051
1085
  { short offcodeNCount[MaxOff+1];
1052
1086
  unsigned offcodeMaxValue = MaxOff, offcodeLog;
1053
1087
  size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
1054
- RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted);
1055
- RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted);
1056
- RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted);
1088
+ RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
1089
+ RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
1090
+ RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
1057
1091
  ZSTD_buildFSETable( entropy->OFTable,
1058
1092
  offcodeNCount, offcodeMaxValue,
1059
1093
  OF_base, OF_bits,
@@ -1064,9 +1098,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1064
1098
  { short matchlengthNCount[MaxML+1];
1065
1099
  unsigned matchlengthMaxValue = MaxML, matchlengthLog;
1066
1100
  size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
1067
- RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted);
1068
- RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted);
1069
- RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted);
1101
+ RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
1102
+ RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
1103
+ RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
1070
1104
  ZSTD_buildFSETable( entropy->MLTable,
1071
1105
  matchlengthNCount, matchlengthMaxValue,
1072
1106
  ML_base, ML_bits,
@@ -1077,9 +1111,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1077
1111
  { short litlengthNCount[MaxLL+1];
1078
1112
  unsigned litlengthMaxValue = MaxLL, litlengthLog;
1079
1113
  size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
1080
- RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted);
1081
- RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted);
1082
- RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted);
1114
+ RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
1115
+ RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
1116
+ RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
1083
1117
  ZSTD_buildFSETable( entropy->LLTable,
1084
1118
  litlengthNCount, litlengthMaxValue,
1085
1119
  LL_base, LL_bits,
@@ -1087,13 +1121,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1087
1121
  dictPtr += litlengthHeaderSize;
1088
1122
  }
1089
1123
 
1090
- RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted);
1124
+ RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
1091
1125
  { int i;
1092
1126
  size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
1093
1127
  for (i=0; i<3; i++) {
1094
1128
  U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
1095
- RETURN_ERROR_IF(rep==0 || rep >= dictContentSize,
1096
- dictionary_corrupted);
1129
+ RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
1130
+ dictionary_corrupted, "");
1097
1131
  entropy->rep[i] = rep;
1098
1132
  } }
1099
1133
 
@@ -1111,7 +1145,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
1111
1145
 
1112
1146
  /* load entropy tables */
1113
1147
  { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
1114
- RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted);
1148
+ RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
1115
1149
  dict = (const char*)dict + eSize;
1116
1150
  dictSize -= eSize;
1117
1151
  }
@@ -1134,6 +1168,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
1134
1168
  dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
1135
1169
  dctx->litEntropy = dctx->fseEntropy = 0;
1136
1170
  dctx->dictID = 0;
1171
+ dctx->bType = bt_reserved;
1137
1172
  ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
1138
1173
  memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
1139
1174
  dctx->LLTptr = dctx->entropy.LLTable;
@@ -1145,11 +1180,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
1145
1180
 
1146
1181
  size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
1147
1182
  {
1148
- FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) );
1183
+ FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
1149
1184
  if (dict && dictSize)
1150
1185
  RETURN_ERROR_IF(
1151
1186
  ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
1152
- dictionary_corrupted);
1187
+ dictionary_corrupted, "");
1153
1188
  return 0;
1154
1189
  }
1155
1190
 
@@ -1168,7 +1203,7 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
1168
1203
  DEBUGLOG(4, "DDict is %s",
1169
1204
  dctx->ddictIsCold ? "~cold~" : "hot!");
1170
1205
  }
1171
- FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) );
1206
+ FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
1172
1207
  if (ddict) { /* NULL ddict is equivalent to no dictionary */
1173
1208
  ZSTD_copyDDictParameters(dctx, ddict);
1174
1209
  }
@@ -1259,11 +1294,11 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
1259
1294
  ZSTD_dictLoadMethod_e dictLoadMethod,
1260
1295
  ZSTD_dictContentType_e dictContentType)
1261
1296
  {
1262
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
1297
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1263
1298
  ZSTD_clearDict(dctx);
1264
- if (dict && dictSize >= 8) {
1299
+ if (dict && dictSize != 0) {
1265
1300
  dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
1266
- RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
1301
+ RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
1267
1302
  dctx->ddict = dctx->ddictLocal;
1268
1303
  dctx->dictUses = ZSTD_use_indefinitely;
1269
1304
  }
@@ -1282,7 +1317,7 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
1282
1317
 
1283
1318
  size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
1284
1319
  {
1285
- FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType));
1320
+ FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
1286
1321
  dctx->dictUses = ZSTD_use_once;
1287
1322
  return 0;
1288
1323
  }
@@ -1294,14 +1329,14 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
1294
1329
 
1295
1330
 
1296
1331
  /* ZSTD_initDStream_usingDict() :
1297
- * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
1332
+ * return : expected size, aka ZSTD_startingInputLength().
1298
1333
  * this function cannot fail */
1299
1334
  size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
1300
1335
  {
1301
1336
  DEBUGLOG(4, "ZSTD_initDStream_usingDict");
1302
- FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) );
1303
- FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
1304
- return ZSTD_FRAMEHEADERSIZE_PREFIX;
1337
+ FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
1338
+ FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
1339
+ return ZSTD_startingInputLength(zds->format);
1305
1340
  }
1306
1341
 
1307
1342
  /* note : this variant can't fail */
@@ -1316,24 +1351,24 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
1316
1351
  * this function cannot fail */
1317
1352
  size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
1318
1353
  {
1319
- FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) );
1320
- FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) );
1321
- return ZSTD_FRAMEHEADERSIZE_PREFIX;
1354
+ FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
1355
+ FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
1356
+ return ZSTD_startingInputLength(dctx->format);
1322
1357
  }
1323
1358
 
1324
1359
  /* ZSTD_resetDStream() :
1325
- * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
1360
+ * return : expected size, aka ZSTD_startingInputLength().
1326
1361
  * this function cannot fail */
1327
1362
  size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
1328
1363
  {
1329
- FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only));
1330
- return ZSTD_FRAMEHEADERSIZE_PREFIX;
1364
+ FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
1365
+ return ZSTD_startingInputLength(dctx->format);
1331
1366
  }
1332
1367
 
1333
1368
 
1334
1369
  size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
1335
1370
  {
1336
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
1371
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1337
1372
  ZSTD_clearDict(dctx);
1338
1373
  if (ddict) {
1339
1374
  dctx->ddict = ddict;
@@ -1350,9 +1385,9 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
1350
1385
  ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
1351
1386
  size_t const min = (size_t)1 << bounds.lowerBound;
1352
1387
  size_t const max = (size_t)1 << bounds.upperBound;
1353
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
1354
- RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound);
1355
- RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound);
1388
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1389
+ RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
1390
+ RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
1356
1391
  dctx->maxWindowSize = maxWindowSize;
1357
1392
  return 0;
1358
1393
  }
@@ -1375,6 +1410,10 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
1375
1410
  bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
1376
1411
  ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
1377
1412
  return bounds;
1413
+ case ZSTD_d_stableOutBuffer:
1414
+ bounds.lowerBound = (int)ZSTD_obm_buffered;
1415
+ bounds.upperBound = (int)ZSTD_obm_stable;
1416
+ return bounds;
1378
1417
  default:;
1379
1418
  }
1380
1419
  bounds.error = ERROR(parameter_unsupported);
@@ -1394,12 +1433,12 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
1394
1433
  }
1395
1434
 
1396
1435
  #define CHECK_DBOUNDS(p,v) { \
1397
- RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound); \
1436
+ RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
1398
1437
  }
1399
1438
 
1400
1439
  size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
1401
1440
  {
1402
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
1441
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1403
1442
  switch(dParam) {
1404
1443
  case ZSTD_d_windowLogMax:
1405
1444
  if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
@@ -1410,9 +1449,13 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
1410
1449
  CHECK_DBOUNDS(ZSTD_d_format, value);
1411
1450
  dctx->format = (ZSTD_format_e)value;
1412
1451
  return 0;
1452
+ case ZSTD_d_stableOutBuffer:
1453
+ CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
1454
+ dctx->outBufferMode = (ZSTD_outBufferMode_e)value;
1455
+ return 0;
1413
1456
  default:;
1414
1457
  }
1415
- RETURN_ERROR(parameter_unsupported);
1458
+ RETURN_ERROR(parameter_unsupported, "");
1416
1459
  }
1417
1460
 
1418
1461
  size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
@@ -1424,7 +1467,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
1424
1467
  }
1425
1468
  if ( (reset == ZSTD_reset_parameters)
1426
1469
  || (reset == ZSTD_reset_session_and_parameters) ) {
1427
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
1470
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1428
1471
  ZSTD_clearDict(dctx);
1429
1472
  dctx->format = ZSTD_f_zstd1;
1430
1473
  dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
@@ -1445,7 +1488,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
1445
1488
  unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
1446
1489
  size_t const minRBSize = (size_t) neededSize;
1447
1490
  RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
1448
- frameParameter_windowTooLarge);
1491
+ frameParameter_windowTooLarge, "");
1449
1492
  return minRBSize;
1450
1493
  }
1451
1494
 
@@ -1463,30 +1506,94 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
1463
1506
  ZSTD_frameHeader zfh;
1464
1507
  size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
1465
1508
  if (ZSTD_isError(err)) return err;
1466
- RETURN_ERROR_IF(err>0, srcSize_wrong);
1509
+ RETURN_ERROR_IF(err>0, srcSize_wrong, "");
1467
1510
  RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
1468
- frameParameter_windowTooLarge);
1511
+ frameParameter_windowTooLarge, "");
1469
1512
  return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
1470
1513
  }
1471
1514
 
1472
1515
 
1473
1516
  /* ***** Decompression ***** */
1474
1517
 
1475
- MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
1518
+ static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
1519
+ {
1520
+ return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
1521
+ }
1522
+
1523
+ static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
1524
+ {
1525
+ if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
1526
+ zds->oversizedDuration++;
1527
+ else
1528
+ zds->oversizedDuration = 0;
1529
+ }
1530
+
1531
+ static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
1532
+ {
1533
+ return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
1534
+ }
1535
+
1536
+ /* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
1537
+ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
1476
1538
  {
1477
- size_t const length = MIN(dstCapacity, srcSize);
1478
- memcpy(dst, src, length);
1479
- return length;
1539
+ ZSTD_outBuffer const expect = zds->expectedOutBuffer;
1540
+ /* No requirement when ZSTD_obm_stable is not enabled. */
1541
+ if (zds->outBufferMode != ZSTD_obm_stable)
1542
+ return 0;
1543
+ /* Any buffer is allowed in zdss_init, this must be the same for every other call until
1544
+ * the context is reset.
1545
+ */
1546
+ if (zds->streamStage == zdss_init)
1547
+ return 0;
1548
+ /* The buffer must match our expectation exactly. */
1549
+ if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
1550
+ return 0;
1551
+ RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!");
1480
1552
  }
1481
1553
 
1554
+ /* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
1555
+ * and updates the stage and the output buffer state. This call is extracted so it can be
1556
+ * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
1557
+ * NOTE: You must break after calling this function since the streamStage is modified.
1558
+ */
1559
+ static size_t ZSTD_decompressContinueStream(
1560
+ ZSTD_DStream* zds, char** op, char* oend,
1561
+ void const* src, size_t srcSize) {
1562
+ int const isSkipFrame = ZSTD_isSkipFrame(zds);
1563
+ if (zds->outBufferMode == ZSTD_obm_buffered) {
1564
+ size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
1565
+ size_t const decodedSize = ZSTD_decompressContinue(zds,
1566
+ zds->outBuff + zds->outStart, dstSize, src, srcSize);
1567
+ FORWARD_IF_ERROR(decodedSize, "");
1568
+ if (!decodedSize && !isSkipFrame) {
1569
+ zds->streamStage = zdss_read;
1570
+ } else {
1571
+ zds->outEnd = zds->outStart + decodedSize;
1572
+ zds->streamStage = zdss_flush;
1573
+ }
1574
+ } else {
1575
+ /* Write directly into the output buffer */
1576
+ size_t const dstSize = isSkipFrame ? 0 : oend - *op;
1577
+ size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
1578
+ FORWARD_IF_ERROR(decodedSize, "");
1579
+ *op += decodedSize;
1580
+ /* Flushing is not needed. */
1581
+ zds->streamStage = zdss_read;
1582
+ assert(*op <= oend);
1583
+ assert(zds->outBufferMode == ZSTD_obm_stable);
1584
+ }
1585
+ return 0;
1586
+ }
1482
1587
 
1483
1588
  size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
1484
1589
  {
1485
- const char* const istart = (const char*)(input->src) + input->pos;
1486
- const char* const iend = (const char*)(input->src) + input->size;
1590
+ const char* const src = (const char*)input->src;
1591
+ const char* const istart = input->pos != 0 ? src + input->pos : src;
1592
+ const char* const iend = input->size != 0 ? src + input->size : src;
1487
1593
  const char* ip = istart;
1488
- char* const ostart = (char*)(output->dst) + output->pos;
1489
- char* const oend = (char*)(output->dst) + output->size;
1594
+ char* const dst = (char*)output->dst;
1595
+ char* const ostart = output->pos != 0 ? dst + output->pos : dst;
1596
+ char* const oend = output->size != 0 ? dst + output->size : dst;
1490
1597
  char* op = ostart;
1491
1598
  U32 someMoreWork = 1;
1492
1599
 
@@ -1502,6 +1609,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1502
1609
  "forbidden. out: pos: %u vs size: %u",
1503
1610
  (U32)output->pos, (U32)output->size);
1504
1611
  DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
1612
+ FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
1505
1613
 
1506
1614
  while (someMoreWork) {
1507
1615
  switch(zds->streamStage)
@@ -1512,6 +1620,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1512
1620
  zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
1513
1621
  zds->legacyVersion = 0;
1514
1622
  zds->hostageByte = 0;
1623
+ zds->expectedOutBuffer = *output;
1515
1624
  /* fall-through */
1516
1625
 
1517
1626
  case zdss_loadHeader :
@@ -1539,7 +1648,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1539
1648
  "legacy support is incompatible with static dctx");
1540
1649
  FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
1541
1650
  zds->previousLegacyVersion, legacyVersion,
1542
- dict, dictSize));
1651
+ dict, dictSize), "");
1543
1652
  zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
1544
1653
  { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
1545
1654
  if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
@@ -1558,7 +1667,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1558
1667
  zds->lhSize += remainingInput;
1559
1668
  }
1560
1669
  input->pos = input->size;
1561
- return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
1670
+ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
1562
1671
  }
1563
1672
  assert(ip != NULL);
1564
1673
  memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
@@ -1566,7 +1675,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1566
1675
  } }
1567
1676
 
1568
1677
  /* check for single-pass mode opportunity */
1569
- if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
1678
+ if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
1679
+ && zds->fParams.frameType != ZSTD_skippableFrame
1570
1680
  && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
1571
1681
  size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
1572
1682
  if (cSize <= (size_t)(iend-istart)) {
@@ -1582,15 +1692,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1582
1692
  break;
1583
1693
  } }
1584
1694
 
1695
+ /* Check output buffer is large enough for ZSTD_odm_stable. */
1696
+ if (zds->outBufferMode == ZSTD_obm_stable
1697
+ && zds->fParams.frameType != ZSTD_skippableFrame
1698
+ && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
1699
+ && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
1700
+ RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
1701
+ }
1702
+
1585
1703
  /* Consume header (see ZSTDds_decodeFrameHeader) */
1586
1704
  DEBUGLOG(4, "Consume header");
1587
- FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)));
1705
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
1588
1706
 
1589
1707
  if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
1590
1708
  zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
1591
1709
  zds->stage = ZSTDds_skipFrame;
1592
1710
  } else {
1593
- FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
1711
+ FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
1594
1712
  zds->expected = ZSTD_blockHeaderSize;
1595
1713
  zds->stage = ZSTDds_decodeBlockHeader;
1596
1714
  }
@@ -1601,40 +1719,48 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1601
1719
  (U32)(zds->maxWindowSize >> 10) );
1602
1720
  zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
1603
1721
  RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
1604
- frameParameter_windowTooLarge);
1722
+ frameParameter_windowTooLarge, "");
1605
1723
 
1606
1724
  /* Adapt buffer sizes to frame header instructions */
1607
1725
  { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
1608
- size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize);
1609
- if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) {
1610
- size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
1611
- DEBUGLOG(4, "inBuff : from %u to %u",
1612
- (U32)zds->inBuffSize, (U32)neededInBuffSize);
1613
- DEBUGLOG(4, "outBuff : from %u to %u",
1614
- (U32)zds->outBuffSize, (U32)neededOutBuffSize);
1615
- if (zds->staticSize) { /* static DCtx */
1616
- DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
1617
- assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
1618
- RETURN_ERROR_IF(
1619
- bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
1620
- memory_allocation);
1621
- } else {
1622
- ZSTD_free(zds->inBuff, zds->customMem);
1623
- zds->inBuffSize = 0;
1624
- zds->outBuffSize = 0;
1625
- zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
1626
- RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation);
1627
- }
1628
- zds->inBuffSize = neededInBuffSize;
1629
- zds->outBuff = zds->inBuff + zds->inBuffSize;
1630
- zds->outBuffSize = neededOutBuffSize;
1631
- } }
1726
+ size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered
1727
+ ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
1728
+ : 0;
1729
+
1730
+ ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
1731
+
1732
+ { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
1733
+ int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
1734
+
1735
+ if (tooSmall || tooLarge) {
1736
+ size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
1737
+ DEBUGLOG(4, "inBuff : from %u to %u",
1738
+ (U32)zds->inBuffSize, (U32)neededInBuffSize);
1739
+ DEBUGLOG(4, "outBuff : from %u to %u",
1740
+ (U32)zds->outBuffSize, (U32)neededOutBuffSize);
1741
+ if (zds->staticSize) { /* static DCtx */
1742
+ DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
1743
+ assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
1744
+ RETURN_ERROR_IF(
1745
+ bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
1746
+ memory_allocation, "");
1747
+ } else {
1748
+ ZSTD_free(zds->inBuff, zds->customMem);
1749
+ zds->inBuffSize = 0;
1750
+ zds->outBuffSize = 0;
1751
+ zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
1752
+ RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
1753
+ }
1754
+ zds->inBuffSize = neededInBuffSize;
1755
+ zds->outBuff = zds->inBuff + zds->inBuffSize;
1756
+ zds->outBuffSize = neededOutBuffSize;
1757
+ } } }
1632
1758
  zds->streamStage = zdss_read;
1633
1759
  /* fall-through */
1634
1760
 
1635
1761
  case zdss_read:
1636
1762
  DEBUGLOG(5, "stage zdss_read");
1637
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
1763
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip);
1638
1764
  DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
1639
1765
  if (neededInSize==0) { /* end of frame */
1640
1766
  zds->streamStage = zdss_init;
@@ -1642,15 +1768,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1642
1768
  break;
1643
1769
  }
1644
1770
  if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
1645
- int const isSkipFrame = ZSTD_isSkipFrame(zds);
1646
- size_t const decodedSize = ZSTD_decompressContinue(zds,
1647
- zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
1648
- ip, neededInSize);
1649
- if (ZSTD_isError(decodedSize)) return decodedSize;
1771
+ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
1650
1772
  ip += neededInSize;
1651
- if (!decodedSize && !isSkipFrame) break; /* this was just a header */
1652
- zds->outEnd = zds->outStart + decodedSize;
1653
- zds->streamStage = zdss_flush;
1773
+ /* Function modifies the stage so we must break */
1654
1774
  break;
1655
1775
  } }
1656
1776
  if (ip==iend) { someMoreWork = 0; break; } /* no more input */
@@ -1662,6 +1782,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1662
1782
  size_t const toLoad = neededInSize - zds->inPos;
1663
1783
  int const isSkipFrame = ZSTD_isSkipFrame(zds);
1664
1784
  size_t loadedSize;
1785
+ /* At this point we shouldn't be decompressing a block that we can stream. */
1786
+ assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
1665
1787
  if (isSkipFrame) {
1666
1788
  loadedSize = MIN(toLoad, (size_t)(iend-ip));
1667
1789
  } else {
@@ -1675,17 +1797,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1675
1797
  if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
1676
1798
 
1677
1799
  /* decode loaded input */
1678
- { size_t const decodedSize = ZSTD_decompressContinue(zds,
1679
- zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
1680
- zds->inBuff, neededInSize);
1681
- if (ZSTD_isError(decodedSize)) return decodedSize;
1682
- zds->inPos = 0; /* input is consumed */
1683
- if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */
1684
- zds->outEnd = zds->outStart + decodedSize;
1685
- } }
1686
- zds->streamStage = zdss_flush;
1687
- /* fall-through */
1688
-
1800
+ zds->inPos = 0; /* input is consumed */
1801
+ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
1802
+ /* Function modifies the stage so we must break */
1803
+ break;
1804
+ }
1689
1805
  case zdss_flush:
1690
1806
  { size_t const toFlushSize = zds->outEnd - zds->outStart;
1691
1807
  size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
@@ -1708,17 +1824,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1708
1824
 
1709
1825
  default:
1710
1826
  assert(0); /* impossible */
1711
- RETURN_ERROR(GENERIC); /* some compiler require default to do something */
1827
+ RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
1712
1828
  } }
1713
1829
 
1714
1830
  /* result */
1715
1831
  input->pos = (size_t)(ip - (const char*)(input->src));
1716
1832
  output->pos = (size_t)(op - (char*)(output->dst));
1833
+
1834
+ /* Update the expected output buffer for ZSTD_obm_stable. */
1835
+ zds->expectedOutBuffer = *output;
1836
+
1717
1837
  if ((ip==istart) && (op==ostart)) { /* no forward progress */
1718
1838
  zds->noForwardProgress ++;
1719
1839
  if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
1720
- RETURN_ERROR_IF(op==oend, dstSize_tooSmall);
1721
- RETURN_ERROR_IF(ip==iend, srcSize_wrong);
1840
+ RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
1841
+ RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
1722
1842
  assert(0);
1723
1843
  }
1724
1844
  } else {