zstd-ruby 1.4.0.0 → 1.4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +5 -0
  4. data/ext/zstdruby/libzstd/common/compiler.h +7 -0
  5. data/ext/zstdruby/libzstd/common/zstd_internal.h +58 -6
  6. data/ext/zstdruby/libzstd/compress/zstd_compress.c +175 -117
  7. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +74 -30
  8. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +56 -36
  9. data/ext/zstdruby/libzstd/compress/zstd_fast.c +35 -14
  10. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +10 -5
  11. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +1 -1
  12. data/ext/zstdruby/libzstd/compress/zstd_opt.c +45 -32
  13. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +18 -7
  14. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +1 -0
  15. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +12 -9
  16. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +20 -9
  17. data/ext/zstdruby/libzstd/dictBuilder/cover.c +154 -43
  18. data/ext/zstdruby/libzstd/dictBuilder/cover.h +38 -3
  19. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +46 -39
  20. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +9 -9
  21. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +5 -0
  22. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -0
  23. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +95 -101
  24. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +11 -6
  25. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +11 -6
  26. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +11 -8
  27. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +88 -84
  28. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +2 -4
  29. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +2 -4
  30. data/ext/zstdruby/libzstd/zstd.h +53 -21
  31. data/lib/zstd-ruby/version.rb +1 -1
  32. metadata +3 -4
@@ -741,7 +741,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
741
741
  /* analyze, build stats, starting with literals */
742
742
  { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
743
743
  if (HUF_isError(maxNbBits)) {
744
- eSize = ERROR(GENERIC);
744
+ eSize = maxNbBits;
745
745
  DISPLAYLEVEL(1, " HUF_buildCTable error \n");
746
746
  goto _cleanup;
747
747
  }
@@ -764,7 +764,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
764
764
  total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
765
765
  errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
766
766
  if (FSE_isError(errorCode)) {
767
- eSize = ERROR(GENERIC);
767
+ eSize = errorCode;
768
768
  DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
769
769
  goto _cleanup;
770
770
  }
@@ -773,7 +773,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
773
773
  total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
774
774
  errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
775
775
  if (FSE_isError(errorCode)) {
776
- eSize = ERROR(GENERIC);
776
+ eSize = errorCode;
777
777
  DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
778
778
  goto _cleanup;
779
779
  }
@@ -782,7 +782,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
782
782
  total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
783
783
  errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
784
784
  if (FSE_isError(errorCode)) {
785
- eSize = ERROR(GENERIC);
785
+ eSize = errorCode;
786
786
  DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
787
787
  goto _cleanup;
788
788
  }
@@ -791,7 +791,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
791
791
  /* write result to buffer */
792
792
  { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
793
793
  if (HUF_isError(hhSize)) {
794
- eSize = ERROR(GENERIC);
794
+ eSize = hhSize;
795
795
  DISPLAYLEVEL(1, "HUF_writeCTable error \n");
796
796
  goto _cleanup;
797
797
  }
@@ -802,7 +802,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
802
802
 
803
803
  { size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog);
804
804
  if (FSE_isError(ohSize)) {
805
- eSize = ERROR(GENERIC);
805
+ eSize = ohSize;
806
806
  DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount \n");
807
807
  goto _cleanup;
808
808
  }
@@ -813,7 +813,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
813
813
 
814
814
  { size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog);
815
815
  if (FSE_isError(mhSize)) {
816
- eSize = ERROR(GENERIC);
816
+ eSize = mhSize;
817
817
  DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount \n");
818
818
  goto _cleanup;
819
819
  }
@@ -824,7 +824,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
824
824
 
825
825
  { size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog);
826
826
  if (FSE_isError(lhSize)) {
827
- eSize = ERROR(GENERIC);
827
+ eSize = lhSize;
828
828
  DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount \n");
829
829
  goto _cleanup;
830
830
  }
@@ -834,7 +834,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
834
834
  }
835
835
 
836
836
  if (maxDstSize<12) {
837
- eSize = ERROR(GENERIC);
837
+ eSize = ERROR(dstSize_tooSmall);
838
838
  DISPLAYLEVEL(1, "not enough space to write RepOffsets \n");
839
839
  goto _cleanup;
840
840
  }
@@ -94,6 +94,8 @@ typedef struct {
94
94
  unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
95
95
  unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
96
96
  double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
97
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
98
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
97
99
  ZDICT_params_t zParams;
98
100
  } ZDICT_cover_params_t;
99
101
 
@@ -105,6 +107,9 @@ typedef struct {
105
107
  unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
106
108
  double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
107
109
  unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
110
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
111
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
112
+
108
113
  ZDICT_params_t zParams;
109
114
  } ZDICT_fastCover_params_t;
110
115
 
@@ -238,6 +238,10 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
238
238
  frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
239
239
  break;
240
240
  }
241
+ if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {
242
+ frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
243
+ frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
244
+ }
241
245
  return frameSizeInfo;
242
246
  }
243
247
 
@@ -1073,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1073
1073
  const void* cSrc, size_t cSrcSize,
1074
1074
  const U16* DTable)
1075
1075
  {
1076
- BYTE* const ostart = (BYTE*) dst;
1077
- BYTE* op = ostart;
1078
- BYTE* const omax = op + maxDstSize;
1079
- BYTE* const olimit = omax-15;
1080
-
1081
- const void* ptr = DTable;
1082
- const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
1083
- const U32 dtLog = DTable[0];
1084
- size_t errorCode;
1085
- U32 reloadStatus;
1086
-
1087
- /* Init */
1088
-
1089
- const U16* jumpTable = (const U16*)cSrc;
1090
- const size_t length1 = FSE_readLE16(jumpTable);
1091
- const size_t length2 = FSE_readLE16(jumpTable+1);
1092
- const size_t length3 = FSE_readLE16(jumpTable+2);
1093
- const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
1094
- const char* const start1 = (const char*)(cSrc) + 6;
1095
- const char* const start2 = start1 + length1;
1096
- const char* const start3 = start2 + length2;
1097
- const char* const start4 = start3 + length3;
1098
- FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
1099
-
1100
- if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
1101
-
1102
- errorCode = FSE_initDStream(&bitD1, start1, length1);
1103
- if (FSE_isError(errorCode)) return errorCode;
1104
- errorCode = FSE_initDStream(&bitD2, start2, length2);
1105
- if (FSE_isError(errorCode)) return errorCode;
1106
- errorCode = FSE_initDStream(&bitD3, start3, length3);
1107
- if (FSE_isError(errorCode)) return errorCode;
1108
- errorCode = FSE_initDStream(&bitD4, start4, length4);
1109
- if (FSE_isError(errorCode)) return errorCode;
1110
-
1111
- reloadStatus=FSE_reloadDStream(&bitD2);
1112
-
1113
- /* 16 symbols per loop */
1114
- for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
1115
- op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
1076
+ if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
1116
1077
  {
1117
- #define HUF_DECODE_SYMBOL_0(n, Dstream) \
1118
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
1119
-
1120
- #define HUF_DECODE_SYMBOL_1(n, Dstream) \
1121
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1122
- if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
1123
-
1124
- #define HUF_DECODE_SYMBOL_2(n, Dstream) \
1125
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1126
- if (FSE_32bits()) FSE_reloadDStream(&Dstream)
1127
-
1128
- HUF_DECODE_SYMBOL_1( 0, bitD1);
1129
- HUF_DECODE_SYMBOL_1( 1, bitD2);
1130
- HUF_DECODE_SYMBOL_1( 2, bitD3);
1131
- HUF_DECODE_SYMBOL_1( 3, bitD4);
1132
- HUF_DECODE_SYMBOL_2( 4, bitD1);
1133
- HUF_DECODE_SYMBOL_2( 5, bitD2);
1134
- HUF_DECODE_SYMBOL_2( 6, bitD3);
1135
- HUF_DECODE_SYMBOL_2( 7, bitD4);
1136
- HUF_DECODE_SYMBOL_1( 8, bitD1);
1137
- HUF_DECODE_SYMBOL_1( 9, bitD2);
1138
- HUF_DECODE_SYMBOL_1(10, bitD3);
1139
- HUF_DECODE_SYMBOL_1(11, bitD4);
1140
- HUF_DECODE_SYMBOL_0(12, bitD1);
1141
- HUF_DECODE_SYMBOL_0(13, bitD2);
1142
- HUF_DECODE_SYMBOL_0(14, bitD3);
1143
- HUF_DECODE_SYMBOL_0(15, bitD4);
1144
- }
1078
+ BYTE* const ostart = (BYTE*) dst;
1079
+ BYTE* op = ostart;
1080
+ BYTE* const omax = op + maxDstSize;
1081
+ BYTE* const olimit = omax-15;
1082
+
1083
+ const void* ptr = DTable;
1084
+ const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
1085
+ const U32 dtLog = DTable[0];
1086
+ size_t errorCode;
1087
+ U32 reloadStatus;
1088
+
1089
+ /* Init */
1090
+
1091
+ const U16* jumpTable = (const U16*)cSrc;
1092
+ const size_t length1 = FSE_readLE16(jumpTable);
1093
+ const size_t length2 = FSE_readLE16(jumpTable+1);
1094
+ const size_t length3 = FSE_readLE16(jumpTable+2);
1095
+ const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
1096
+ const char* const start1 = (const char*)(cSrc) + 6;
1097
+ const char* const start2 = start1 + length1;
1098
+ const char* const start3 = start2 + length2;
1099
+ const char* const start4 = start3 + length3;
1100
+ FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
1101
+
1102
+ if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
1103
+
1104
+ errorCode = FSE_initDStream(&bitD1, start1, length1);
1105
+ if (FSE_isError(errorCode)) return errorCode;
1106
+ errorCode = FSE_initDStream(&bitD2, start2, length2);
1107
+ if (FSE_isError(errorCode)) return errorCode;
1108
+ errorCode = FSE_initDStream(&bitD3, start3, length3);
1109
+ if (FSE_isError(errorCode)) return errorCode;
1110
+ errorCode = FSE_initDStream(&bitD4, start4, length4);
1111
+ if (FSE_isError(errorCode)) return errorCode;
1112
+
1113
+ reloadStatus=FSE_reloadDStream(&bitD2);
1114
+
1115
+ /* 16 symbols per loop */
1116
+ for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
1117
+ op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
1118
+ {
1119
+ #define HUF_DECODE_SYMBOL_0(n, Dstream) \
1120
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
1121
+
1122
+ #define HUF_DECODE_SYMBOL_1(n, Dstream) \
1123
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1124
+ if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
1125
+
1126
+ #define HUF_DECODE_SYMBOL_2(n, Dstream) \
1127
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1128
+ if (FSE_32bits()) FSE_reloadDStream(&Dstream)
1129
+
1130
+ HUF_DECODE_SYMBOL_1( 0, bitD1);
1131
+ HUF_DECODE_SYMBOL_1( 1, bitD2);
1132
+ HUF_DECODE_SYMBOL_1( 2, bitD3);
1133
+ HUF_DECODE_SYMBOL_1( 3, bitD4);
1134
+ HUF_DECODE_SYMBOL_2( 4, bitD1);
1135
+ HUF_DECODE_SYMBOL_2( 5, bitD2);
1136
+ HUF_DECODE_SYMBOL_2( 6, bitD3);
1137
+ HUF_DECODE_SYMBOL_2( 7, bitD4);
1138
+ HUF_DECODE_SYMBOL_1( 8, bitD1);
1139
+ HUF_DECODE_SYMBOL_1( 9, bitD2);
1140
+ HUF_DECODE_SYMBOL_1(10, bitD3);
1141
+ HUF_DECODE_SYMBOL_1(11, bitD4);
1142
+ HUF_DECODE_SYMBOL_0(12, bitD1);
1143
+ HUF_DECODE_SYMBOL_0(13, bitD2);
1144
+ HUF_DECODE_SYMBOL_0(14, bitD3);
1145
+ HUF_DECODE_SYMBOL_0(15, bitD4);
1146
+ }
1145
1147
 
1146
- if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
1147
- return (size_t)-FSE_ERROR_corruptionDetected;
1148
+ if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
1149
+ return (size_t)-FSE_ERROR_corruptionDetected;
1148
1150
 
1149
- /* tail */
1150
- {
1151
- // bitTail = bitD1; // *much* slower : -20% !??!
1152
- FSE_DStream_t bitTail;
1153
- bitTail.ptr = bitD1.ptr;
1154
- bitTail.bitsConsumed = bitD1.bitsConsumed;
1155
- bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
1156
- bitTail.start = start1;
1157
- for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
1151
+ /* tail */
1158
1152
  {
1159
- HUF_DECODE_SYMBOL_0(0, bitTail);
1160
- }
1153
+ // bitTail = bitD1; // *much* slower : -20% !??!
1154
+ FSE_DStream_t bitTail;
1155
+ bitTail.ptr = bitD1.ptr;
1156
+ bitTail.bitsConsumed = bitD1.bitsConsumed;
1157
+ bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
1158
+ bitTail.start = start1;
1159
+ for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
1160
+ {
1161
+ HUF_DECODE_SYMBOL_0(0, bitTail);
1162
+ }
1161
1163
 
1162
- if (FSE_endOfDStream(&bitTail))
1163
- return op-ostart;
1164
- }
1164
+ if (FSE_endOfDStream(&bitTail))
1165
+ return op-ostart;
1166
+ }
1165
1167
 
1166
- if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
1168
+ if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
1167
1169
 
1168
- return (size_t)-FSE_ERROR_corruptionDetected;
1170
+ return (size_t)-FSE_ERROR_corruptionDetected;
1171
+ }
1169
1172
  }
1170
1173
 
1171
1174
 
@@ -1355,8 +1358,6 @@ static unsigned ZSTD_isLittleEndian(void)
1355
1358
 
1356
1359
  static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
1357
1360
 
1358
- static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
1359
-
1360
1361
  static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
1361
1362
 
1362
1363
  static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
@@ -1381,16 +1382,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
1381
1382
  }
1382
1383
  }
1383
1384
 
1384
-
1385
- static U32 ZSTD_readLE32(const void* memPtr)
1385
+ static U32 ZSTD_readLE24(const void* memPtr)
1386
1386
  {
1387
- if (ZSTD_isLittleEndian())
1388
- return ZSTD_read32(memPtr);
1389
- else
1390
- {
1391
- const BYTE* p = (const BYTE*)memPtr;
1392
- return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
1393
- }
1387
+ return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
1394
1388
  }
1395
1389
 
1396
1390
  static U32 ZSTD_readBE32(const void* memPtr)
@@ -1704,13 +1698,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
1704
1698
  seqState->prevOffset = seq->offset;
1705
1699
  if (litLength == MaxLL)
1706
1700
  {
1707
- U32 add = dumps<de ? *dumps++ : 0;
1701
+ const U32 add = dumps<de ? *dumps++ : 0;
1708
1702
  if (add < 255) litLength += add;
1709
1703
  else
1710
1704
  {
1711
1705
  if (dumps<=(de-3))
1712
1706
  {
1713
- litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
1707
+ litLength = ZSTD_readLE24(dumps);
1714
1708
  dumps += 3;
1715
1709
  }
1716
1710
  }
@@ -1732,13 +1726,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
1732
1726
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
1733
1727
  if (matchLength == MaxML)
1734
1728
  {
1735
- U32 add = dumps<de ? *dumps++ : 0;
1729
+ const U32 add = dumps<de ? *dumps++ : 0;
1736
1730
  if (add < 255) matchLength += add;
1737
1731
  else
1738
1732
  {
1739
1733
  if (dumps<=(de-3))
1740
1734
  {
1741
- matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
1735
+ matchLength = ZSTD_readLE24(dumps);
1742
1736
  dumps += 3;
1743
1737
  }
1744
1738
  }
@@ -217,6 +217,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
217
217
  }
218
218
  }
219
219
 
220
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
221
+ {
222
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
223
+ }
224
+
220
225
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
221
226
  {
222
227
  if (MEM_isLittleEndian())
@@ -3043,11 +3048,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3043
3048
  seqState->prevOffset = seq->offset;
3044
3049
  if (litLength == MaxLL)
3045
3050
  {
3046
- U32 add = *dumps++;
3051
+ const U32 add = dumps<de ? *dumps++ : 0;
3047
3052
  if (add < 255) litLength += add;
3048
- else
3053
+ else if (dumps + 3 <= de)
3049
3054
  {
3050
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3055
+ litLength = MEM_readLE24(dumps);
3051
3056
  dumps += 3;
3052
3057
  }
3053
3058
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3073,11 +3078,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3073
3078
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3074
3079
  if (matchLength == MaxML)
3075
3080
  {
3076
- U32 add = *dumps++;
3081
+ const U32 add = dumps<de ? *dumps++ : 0;
3077
3082
  if (add < 255) matchLength += add;
3078
- else
3083
+ else if (dumps + 3 <= de)
3079
3084
  {
3080
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3085
+ matchLength = MEM_readLE24(dumps);
3081
3086
  dumps += 3;
3082
3087
  }
3083
3088
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -219,6 +219,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
219
219
  }
220
220
  }
221
221
 
222
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
223
+ {
224
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
225
+ }
226
+
222
227
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
223
228
  {
224
229
  if (MEM_isLittleEndian())
@@ -2684,11 +2689,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2684
2689
  seqState->prevOffset = seq->offset;
2685
2690
  if (litLength == MaxLL)
2686
2691
  {
2687
- U32 add = *dumps++;
2692
+ const U32 add = dumps<de ? *dumps++ : 0;
2688
2693
  if (add < 255) litLength += add;
2689
- else
2694
+ else if (dumps + 3 <= de)
2690
2695
  {
2691
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
2696
+ litLength = MEM_readLE24(dumps);
2692
2697
  dumps += 3;
2693
2698
  }
2694
2699
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -2714,11 +2719,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2714
2719
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
2715
2720
  if (matchLength == MaxML)
2716
2721
  {
2717
- U32 add = *dumps++;
2722
+ const U32 add = dumps<de ? *dumps++ : 0;
2718
2723
  if (add < 255) matchLength += add;
2719
- else
2724
+ else if (dumps + 3 <= de)
2720
2725
  {
2721
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
2726
+ matchLength = MEM_readLE24(dumps);
2722
2727
  dumps += 3;
2723
2728
  }
2724
2729
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -189,6 +189,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
189
189
  }
190
190
  }
191
191
 
192
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
193
+ {
194
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
195
+ }
196
+
192
197
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
193
198
  {
194
199
  if (MEM_isLittleEndian())
@@ -2808,13 +2813,12 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2808
2813
  litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
2809
2814
  prevOffset = litLength ? seq->offset : seqState->prevOffset;
2810
2815
  if (litLength == MaxLL) {
2811
- U32 add = *dumps++;
2816
+ const U32 add = dumps<de ? *dumps++ : 0;
2812
2817
  if (add < 255) litLength += add;
2813
- else {
2814
- litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
2818
+ else if (dumps + 3 <= de) {
2819
+ litLength = MEM_readLE24(dumps);
2815
2820
  dumps += 3;
2816
2821
  }
2817
- if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
2818
2822
  if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
2819
2823
  }
2820
2824
 
@@ -2837,13 +2841,12 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2837
2841
  /* MatchLength */
2838
2842
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
2839
2843
  if (matchLength == MaxML) {
2840
- U32 add = *dumps++;
2844
+ const U32 add = dumps<de ? *dumps++ : 0;
2841
2845
  if (add < 255) matchLength += add;
2842
- else {
2843
- matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
2846
+ else if (dumps + 3 <= de){
2847
+ matchLength = MEM_readLE24(dumps);
2844
2848
  dumps += 3;
2845
2849
  }
2846
- if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
2847
2850
  if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
2848
2851
  }
2849
2852
  matchLength += MINMATCH;