zstd-ruby 1.5.4.0 → 1.5.5.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -11,6 +11,7 @@
11
11
  /*-*************************************
12
12
  * Dependencies
13
13
  ***************************************/
14
+ #include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
14
15
  #include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
15
16
  #include "../common/mem.h"
16
17
  #include "hist.h" /* HIST_countFast_wksp */
@@ -26,7 +27,7 @@
26
27
  #include "zstd_opt.h"
27
28
  #include "zstd_ldm.h"
28
29
  #include "zstd_compress_superblock.h"
29
- #include "../common/bits.h" /* ZSTD_highbit32 */
30
+ #include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */
30
31
 
31
32
  /* ***************************************************************
32
33
  * Tuning parameters
@@ -1177,16 +1178,39 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
1177
1178
 
1178
1179
  size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
1179
1180
  {
1181
+ ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */);
1180
1182
  DEBUGLOG(4, "ZSTD_CCtx_setCParams");
1181
- assert(cctx != NULL);
1182
- if (cctx->streamStage != zcss_init) {
1183
- /* All parameters in @cparams are allowed to be updated during MT compression.
1184
- * This must be signaled, so that MT compression picks up the changes */
1185
- cctx->cParamsChanged = 1;
1186
- }
1187
- /* only update if parameters are valid */
1183
+ /* only update if all parameters are valid */
1188
1184
  FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
1189
- cctx->requestedParams.cParams = cparams;
1185
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), "");
1186
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), "");
1187
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), "");
1188
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), "");
1189
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), "");
1190
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), "");
1191
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), "");
1192
+ return 0;
1193
+ }
1194
+
1195
+ size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams)
1196
+ {
1197
+ ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */);
1198
+ DEBUGLOG(4, "ZSTD_CCtx_setFParams");
1199
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), "");
1200
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), "");
1201
+ FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), "");
1202
+ return 0;
1203
+ }
1204
+
1205
+ size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params)
1206
+ {
1207
+ DEBUGLOG(4, "ZSTD_CCtx_setParams");
1208
+ /* First check cParams, because we want to update all or none. */
1209
+ FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
1210
+ /* Next set fParams, because this could fail if the cctx isn't in init stage. */
1211
+ FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), "");
1212
+ /* Finally set cParams, which should succeed. */
1213
+ FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), "");
1190
1214
  return 0;
1191
1215
  }
1192
1216
 
@@ -1208,9 +1232,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams(
1208
1232
  ZSTD_compressionParameters* cParams);
1209
1233
 
1210
1234
  /**
1211
- * Initializes the local dict using the requested parameters.
1212
- * NOTE: This does not use the pledged src size, because it may be used for more
1213
- * than one compression.
1235
+ * Initializes the local dictionary using requested parameters.
1236
+ * NOTE: Initialization does not employ the pledged src size,
1237
+ * because the dictionary may be used for multiple compressions.
1214
1238
  */
1215
1239
  static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
1216
1240
  {
@@ -1223,8 +1247,8 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
1223
1247
  return 0;
1224
1248
  }
1225
1249
  if (dl->cdict != NULL) {
1226
- assert(cctx->cdict == dl->cdict);
1227
1250
  /* Local dictionary already initialized. */
1251
+ assert(cctx->cdict == dl->cdict);
1228
1252
  return 0;
1229
1253
  }
1230
1254
  assert(dl->dictSize > 0);
@@ -1244,26 +1268,30 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
1244
1268
  }
1245
1269
 
1246
1270
  size_t ZSTD_CCtx_loadDictionary_advanced(
1247
- ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
1248
- ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
1271
+ ZSTD_CCtx* cctx,
1272
+ const void* dict, size_t dictSize,
1273
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1274
+ ZSTD_dictContentType_e dictContentType)
1249
1275
  {
1250
- RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
1251
- "Can't load a dictionary when ctx is not in init stage.");
1252
1276
  DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
1253
- ZSTD_clearAllDicts(cctx); /* in case one already exists */
1254
- if (dict == NULL || dictSize == 0) /* no dictionary mode */
1277
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
1278
+ "Can't load a dictionary when cctx is not in init stage.");
1279
+ ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */
1280
+ if (dict == NULL || dictSize == 0) /* no dictionary */
1255
1281
  return 0;
1256
1282
  if (dictLoadMethod == ZSTD_dlm_byRef) {
1257
1283
  cctx->localDict.dict = dict;
1258
1284
  } else {
1285
+ /* copy dictionary content inside CCtx to own its lifetime */
1259
1286
  void* dictBuffer;
1260
1287
  RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
1261
- "no malloc for static CCtx");
1288
+ "static CCtx can't allocate for an internal copy of dictionary");
1262
1289
  dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
1263
- RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!");
1290
+ RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation,
1291
+ "allocation failed for dictionary content");
1264
1292
  ZSTD_memcpy(dictBuffer, dict, dictSize);
1265
- cctx->localDict.dictBuffer = dictBuffer;
1266
- cctx->localDict.dict = dictBuffer;
1293
+ cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */
1294
+ cctx->localDict.dict = dictBuffer; /* read-only reference */
1267
1295
  }
1268
1296
  cctx->localDict.dictSize = dictSize;
1269
1297
  cctx->localDict.dictContentType = dictContentType;
@@ -1333,7 +1361,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
1333
1361
  if ( (reset == ZSTD_reset_parameters)
1334
1362
  || (reset == ZSTD_reset_session_and_parameters) ) {
1335
1363
  RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
1336
- "Can't reset parameters only when not in init stage.");
1364
+ "Reset parameters is only possible during init stage.");
1337
1365
  ZSTD_clearAllDicts(cctx);
1338
1366
  ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
1339
1367
  return ZSTD_CCtxParams_reset(&cctx->requestedParams);
@@ -1592,7 +1620,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
1592
1620
  + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
1593
1621
  + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
1594
1622
  size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
1595
- ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16))
1623
+ ? ZSTD_cwksp_aligned_alloc_size(hSize)
1596
1624
  : 0;
1597
1625
  size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
1598
1626
  ? optPotentialSpace
@@ -1883,6 +1911,19 @@ typedef enum {
1883
1911
  ZSTD_resetTarget_CCtx
1884
1912
  } ZSTD_resetTarget_e;
1885
1913
 
1914
+ /* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */
1915
+ static U64 ZSTD_bitmix(U64 val, U64 len) {
1916
+ val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24);
1917
+ val *= 0x9FB21C651E98DF25ULL;
1918
+ val ^= (val >> 35) + len ;
1919
+ val *= 0x9FB21C651E98DF25ULL;
1920
+ return val ^ (val >> 28);
1921
+ }
1922
+
1923
+ /* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */
1924
+ static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) {
1925
+ ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4);
1926
+ }
1886
1927
 
1887
1928
  static size_t
1888
1929
  ZSTD_reset_matchState(ZSTD_matchState_t* ms,
@@ -1910,6 +1951,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
1910
1951
  }
1911
1952
 
1912
1953
  ms->hashLog3 = hashLog3;
1954
+ ms->lazySkipping = 0;
1913
1955
 
1914
1956
  ZSTD_invalidateMatchState(ms);
1915
1957
 
@@ -1931,6 +1973,27 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
1931
1973
  ZSTD_cwksp_clean_tables(ws);
1932
1974
  }
1933
1975
 
1976
+ if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
1977
+ /* Row match finder needs an additional table of hashes ("tags") */
1978
+ size_t const tagTableSize = hSize;
1979
+ /* We want to generate a new salt in case we reset a Cctx, but we always want to use
1980
+ * 0 when we reset a Cdict */
1981
+ if(forWho == ZSTD_resetTarget_CCtx) {
1982
+ ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize);
1983
+ ZSTD_advanceHashSalt(ms);
1984
+ } else {
1985
+ /* When we are not salting we want to always memset the memory */
1986
+ ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
1987
+ ZSTD_memset(ms->tagTable, 0, tagTableSize);
1988
+ ms->hashSalt = 0;
1989
+ }
1990
+ { /* Switch to 32-entry rows if searchLog is 5 (or more) */
1991
+ U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
1992
+ assert(cParams->hashLog >= rowLog);
1993
+ ms->rowHashLog = cParams->hashLog - rowLog;
1994
+ }
1995
+ }
1996
+
1934
1997
  /* opt parser space */
1935
1998
  if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
1936
1999
  DEBUGLOG(4, "reserving optimal parser space");
@@ -1942,19 +2005,6 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
1942
2005
  ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
1943
2006
  }
1944
2007
 
1945
- if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
1946
- { /* Row match finder needs an additional table of hashes ("tags") */
1947
- size_t const tagTableSize = hSize*sizeof(U16);
1948
- ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
1949
- if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
1950
- }
1951
- { /* Switch to 32-entry rows if searchLog is 5 (or more) */
1952
- U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
1953
- assert(cParams->hashLog >= rowLog);
1954
- ms->rowHashLog = cParams->hashLog - rowLog;
1955
- }
1956
- }
1957
-
1958
2008
  ms->cParams = *cParams;
1959
2009
 
1960
2010
  RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
@@ -2101,13 +2151,46 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
2101
2151
 
2102
2152
  ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
2103
2153
 
2154
+ FORWARD_IF_ERROR(ZSTD_reset_matchState(
2155
+ &zc->blockState.matchState,
2156
+ ws,
2157
+ &params->cParams,
2158
+ params->useRowMatchFinder,
2159
+ crp,
2160
+ needsIndexReset,
2161
+ ZSTD_resetTarget_CCtx), "");
2162
+
2163
+ zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
2164
+
2165
+ /* ldm hash table */
2166
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
2167
+ /* TODO: avoid memset? */
2168
+ size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
2169
+ zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
2170
+ ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
2171
+ zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
2172
+ zc->maxNbLdmSequences = maxNbLdmSeq;
2173
+
2174
+ ZSTD_window_init(&zc->ldmState.window);
2175
+ zc->ldmState.loadedDictEnd = 0;
2176
+ }
2177
+
2178
+ /* reserve space for block-level external sequences */
2179
+ if (params->useSequenceProducer) {
2180
+ size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
2181
+ zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
2182
+ zc->externalMatchCtx.seqBuffer =
2183
+ (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
2184
+ }
2185
+
2186
+ /* buffers */
2187
+
2104
2188
  /* ZSTD_wildcopy() is used to copy into the literals buffer,
2105
2189
  * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
2106
2190
  */
2107
2191
  zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
2108
2192
  zc->seqStore.maxNbLit = blockSize;
2109
2193
 
2110
- /* buffers */
2111
2194
  zc->bufferedPolicy = zbuff;
2112
2195
  zc->inBuffSize = buffInSize;
2113
2196
  zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
@@ -2130,40 +2213,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
2130
2213
  zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
2131
2214
  zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
2132
2215
  zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
2133
- zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
2134
-
2135
- FORWARD_IF_ERROR(ZSTD_reset_matchState(
2136
- &zc->blockState.matchState,
2137
- ws,
2138
- &params->cParams,
2139
- params->useRowMatchFinder,
2140
- crp,
2141
- needsIndexReset,
2142
- ZSTD_resetTarget_CCtx), "");
2143
-
2144
- /* ldm hash table */
2145
- if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
2146
- /* TODO: avoid memset? */
2147
- size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
2148
- zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
2149
- ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
2150
- zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
2151
- zc->maxNbLdmSequences = maxNbLdmSeq;
2152
-
2153
- ZSTD_window_init(&zc->ldmState.window);
2154
- zc->ldmState.loadedDictEnd = 0;
2155
- }
2156
-
2157
- /* reserve space for block-level external sequences */
2158
- if (params->useSequenceProducer) {
2159
- size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
2160
- zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
2161
- zc->externalMatchCtx.seqBuffer =
2162
- (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
2163
- }
2164
2216
 
2165
2217
  DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
2166
- assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
2218
+ assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace));
2167
2219
 
2168
2220
  zc->initialized = 1;
2169
2221
 
@@ -2338,10 +2390,11 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
2338
2390
  }
2339
2391
  /* copy tag table */
2340
2392
  if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) {
2341
- size_t const tagTableSize = hSize*sizeof(U16);
2393
+ size_t const tagTableSize = hSize;
2342
2394
  ZSTD_memcpy(cctx->blockState.matchState.tagTable,
2343
- cdict->matchState.tagTable,
2344
- tagTableSize);
2395
+ cdict->matchState.tagTable,
2396
+ tagTableSize);
2397
+ cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt;
2345
2398
  }
2346
2399
  }
2347
2400
 
@@ -3858,9 +3911,10 @@ ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRe
3858
3911
  const seqStore_t* const seqStore, U32 const nbSeq)
3859
3912
  {
3860
3913
  U32 idx = 0;
3914
+ U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq;
3861
3915
  for (; idx < nbSeq; ++idx) {
3862
3916
  seqDef* const seq = seqStore->sequencesStart + idx;
3863
- U32 const ll0 = (seq->litLength == 0);
3917
+ U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx);
3864
3918
  U32 const offBase = seq->offBase;
3865
3919
  assert(offBase > 0);
3866
3920
  if (OFFBASE_IS_REPCODE(offBase)) {
@@ -4576,31 +4630,51 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
4576
4630
  }
4577
4631
  }
4578
4632
 
4579
- size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
4580
- void* dst, size_t dstCapacity,
4581
- const void* src, size_t srcSize)
4633
+ size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx,
4634
+ void* dst, size_t dstCapacity,
4635
+ const void* src, size_t srcSize)
4582
4636
  {
4583
4637
  DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize);
4584
4638
  return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
4585
4639
  }
4586
4640
 
4641
+ /* NOTE: Must just wrap ZSTD_compressContinue_public() */
4642
+ size_t ZSTD_compressContinue(ZSTD_CCtx* cctx,
4643
+ void* dst, size_t dstCapacity,
4644
+ const void* src, size_t srcSize)
4645
+ {
4646
+ return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize);
4647
+ }
4587
4648
 
4588
- size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
4649
+ static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx)
4589
4650
  {
4590
4651
  ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
4591
4652
  assert(!ZSTD_checkCParams(cParams));
4592
4653
  return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
4593
4654
  }
4594
4655
 
4595
- size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4656
+ /* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */
4657
+ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
4658
+ {
4659
+ return ZSTD_getBlockSize_deprecated(cctx);
4660
+ }
4661
+
4662
+ /* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */
4663
+ size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4596
4664
  {
4597
4665
  DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
4598
- { size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
4666
+ { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx);
4599
4667
  RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); }
4600
4668
 
4601
4669
  return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
4602
4670
  }
4603
4671
 
4672
+ /* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */
4673
+ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4674
+ {
4675
+ return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize);
4676
+ }
4677
+
4604
4678
  /*! ZSTD_loadDictionaryContent() :
4605
4679
  * @return : 0, or an error code
4606
4680
  */
@@ -4644,31 +4718,42 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
4644
4718
  ip = iend - maxDictSize;
4645
4719
  src = ip;
4646
4720
  srcSize = maxDictSize;
4647
- } }
4721
+ }
4722
+ }
4648
4723
 
4649
4724
  if (srcSize > ZSTD_CHUNKSIZE_MAX) {
4650
4725
  /* We must have cleared our windows when our source is this large. */
4651
4726
  assert(ZSTD_window_isEmpty(ms->window));
4652
4727
  if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window));
4653
4728
  }
4729
+ ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
4654
4730
 
4655
4731
  DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
4656
- ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
4657
- ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
4658
- ms->forceNonContiguous = params->deterministicRefPrefix;
4659
4732
 
4660
- if (loadLdmDict) {
4733
+ if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */
4661
4734
  ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
4662
4735
  ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
4736
+ ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
4737
+ }
4738
+
4739
+ /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */
4740
+ if (params->cParams.strategy < ZSTD_btultra) {
4741
+ U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28);
4742
+ if (srcSize > maxDictSize) {
4743
+ ip = iend - maxDictSize;
4744
+ src = ip;
4745
+ srcSize = maxDictSize;
4746
+ }
4663
4747
  }
4664
4748
 
4749
+ ms->nextToUpdate = (U32)(ip - ms->window.base);
4750
+ ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
4751
+ ms->forceNonContiguous = params->deterministicRefPrefix;
4752
+
4665
4753
  if (srcSize <= HASH_READ_SIZE) return 0;
4666
4754
 
4667
4755
  ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
4668
4756
 
4669
- if (loadLdmDict)
4670
- ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
4671
-
4672
4757
  switch(params->cParams.strategy)
4673
4758
  {
4674
4759
  case ZSTD_fast:
@@ -4688,7 +4773,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
4688
4773
  } else {
4689
4774
  assert(params->useRowMatchFinder != ZSTD_ps_auto);
4690
4775
  if (params->useRowMatchFinder == ZSTD_ps_enable) {
4691
- size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
4776
+ size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog);
4692
4777
  ZSTD_memset(ms->tagTable, 0, tagTableSize);
4693
4778
  ZSTD_row_update(ms, iend-HASH_READ_SIZE);
4694
4779
  DEBUGLOG(4, "Using row-based hash table for lazy dict");
@@ -4991,8 +5076,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
4991
5076
  &cctxParams, pledgedSrcSize);
4992
5077
  }
4993
5078
 
4994
- size_t
4995
- ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
5079
+ static size_t
5080
+ ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
4996
5081
  {
4997
5082
  ZSTD_CCtx_params cctxParams;
4998
5083
  { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
@@ -5003,9 +5088,15 @@ ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
5003
5088
  &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
5004
5089
  }
5005
5090
 
5091
+ size_t
5092
+ ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
5093
+ {
5094
+ return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel);
5095
+ }
5096
+
5006
5097
  size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
5007
5098
  {
5008
- return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
5099
+ return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel);
5009
5100
  }
5010
5101
 
5011
5102
 
@@ -5075,9 +5166,9 @@ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
5075
5166
  #endif
5076
5167
  }
5077
5168
 
5078
- size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
5079
- void* dst, size_t dstCapacity,
5080
- const void* src, size_t srcSize)
5169
+ size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
5170
+ void* dst, size_t dstCapacity,
5171
+ const void* src, size_t srcSize)
5081
5172
  {
5082
5173
  size_t endResult;
5083
5174
  size_t const cSize = ZSTD_compressContinue_internal(cctx,
@@ -5101,6 +5192,14 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
5101
5192
  return cSize + endResult;
5102
5193
  }
5103
5194
 
5195
+ /* NOTE: Must just wrap ZSTD_compressEnd_public() */
5196
+ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx,
5197
+ void* dst, size_t dstCapacity,
5198
+ const void* src, size_t srcSize)
5199
+ {
5200
+ return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
5201
+ }
5202
+
5104
5203
  size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
5105
5204
  void* dst, size_t dstCapacity,
5106
5205
  const void* src, size_t srcSize,
@@ -5129,7 +5228,7 @@ size_t ZSTD_compress_advanced_internal(
5129
5228
  FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
5130
5229
  dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
5131
5230
  params, srcSize, ZSTDb_not_buffered) , "");
5132
- return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
5231
+ return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
5133
5232
  }
5134
5233
 
5135
5234
  size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
@@ -5451,6 +5550,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
5451
5550
  params.cParams = cParams;
5452
5551
  params.useRowMatchFinder = useRowMatchFinder;
5453
5552
  cdict->useRowMatchFinder = useRowMatchFinder;
5553
+ cdict->compressionLevel = ZSTD_NO_CLEVEL;
5454
5554
 
5455
5555
  if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
5456
5556
  dict, dictSize,
@@ -5530,12 +5630,17 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
5530
5630
 
5531
5631
  /* ZSTD_compressBegin_usingCDict() :
5532
5632
  * cdict must be != NULL */
5533
- size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
5633
+ size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
5534
5634
  {
5535
5635
  ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
5536
5636
  return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
5537
5637
  }
5538
5638
 
5639
+ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
5640
+ {
5641
+ return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict);
5642
+ }
5643
+
5539
5644
  /*! ZSTD_compress_usingCDict_internal():
5540
5645
  * Implementation of various ZSTD_compress_usingCDict* functions.
5541
5646
  */
@@ -5545,7 +5650,7 @@ static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx,
5545
5650
  const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
5546
5651
  {
5547
5652
  FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
5548
- return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
5653
+ return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
5549
5654
  }
5550
5655
 
5551
5656
  /*! ZSTD_compress_usingCDict_advanced():
@@ -5803,7 +5908,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
5803
5908
  || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
5804
5909
  && (zcs->inBuffPos == 0) ) {
5805
5910
  /* shortcut to compression pass directly into output buffer */
5806
- size_t const cSize = ZSTD_compressEnd(zcs,
5911
+ size_t const cSize = ZSTD_compressEnd_public(zcs,
5807
5912
  op, oend-op, ip, iend-ip);
5808
5913
  DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
5809
5914
  FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
@@ -5861,9 +5966,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
5861
5966
  if (inputBuffered) {
5862
5967
  unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
5863
5968
  cSize = lastBlock ?
5864
- ZSTD_compressEnd(zcs, cDst, oSize,
5969
+ ZSTD_compressEnd_public(zcs, cDst, oSize,
5865
5970
  zcs->inBuff + zcs->inToCompress, iSize) :
5866
- ZSTD_compressContinue(zcs, cDst, oSize,
5971
+ ZSTD_compressContinue_public(zcs, cDst, oSize,
5867
5972
  zcs->inBuff + zcs->inToCompress, iSize);
5868
5973
  FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
5869
5974
  zcs->frameEnded = lastBlock;
@@ -5879,8 +5984,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
5879
5984
  } else { /* !inputBuffered, hence ZSTD_bm_stable */
5880
5985
  unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
5881
5986
  cSize = lastBlock ?
5882
- ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
5883
- ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
5987
+ ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) :
5988
+ ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize);
5884
5989
  /* Consume the input prior to error checking to mirror buffered mode. */
5885
5990
  if (ip) ip += iSize;
5886
5991
  FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");