zstd-ruby 1.5.1.0 → 1.5.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/README.md +56 -3
- data/Rakefile +8 -2
- data/ext/zstdruby/{zstdruby.h → common.h} +2 -0
- data/ext/zstdruby/extconf.rb +1 -1
- data/ext/zstdruby/libzstd/common/pool.c +11 -6
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/portability_macros.h +6 -0
- data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -4
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +114 -96
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +72 -39
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +10 -10
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +38 -24
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +10 -10
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +11 -11
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +66 -62
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +5 -3
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +66 -43
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +17 -9
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +4 -1
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +2 -2
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +17 -3
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +2 -2
- data/ext/zstdruby/libzstd/zstd.h +1 -1
- data/ext/zstdruby/main.c +14 -0
- data/ext/zstdruby/streaming_compress.c +185 -0
- data/ext/zstdruby/streaming_compress.h +5 -0
- data/ext/zstdruby/streaming_decompress.c +125 -0
- data/ext/zstdruby/zstdruby.c +4 -6
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +1 -1
- metadata +11 -40
- data/.github/dependabot.yml +0 -8
- data/.github/workflows/ruby.yml +0 -35
- data/ext/zstdruby/libzstd/.gitignore +0 -3
- data/ext/zstdruby/libzstd/BUCK +0 -232
- data/ext/zstdruby/libzstd/Makefile +0 -357
- data/ext/zstdruby/libzstd/README.md +0 -217
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +0 -214
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +0 -26
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +0 -167
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +0 -75
- data/ext/zstdruby/libzstd/dll/example/Makefile +0 -48
- data/ext/zstdruby/libzstd/dll/example/README.md +0 -63
- data/ext/zstdruby/libzstd/dll/example/build_package.bat +0 -20
- data/ext/zstdruby/libzstd/dll/example/fullbench-dll.sln +0 -25
- data/ext/zstdruby/libzstd/dll/example/fullbench-dll.vcxproj +0 -181
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +0 -415
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +0 -2158
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +0 -94
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +0 -3518
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +0 -93
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +0 -3160
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +0 -93
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -3647
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +0 -142
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +0 -4050
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +0 -162
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +0 -4154
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +0 -172
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +0 -4541
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +0 -187
- data/ext/zstdruby/libzstd/libzstd.mk +0 -185
- data/ext/zstdruby/libzstd/libzstd.pc.in +0 -16
- data/ext/zstdruby/libzstd/modulemap/module.modulemap +0 -4
@@ -197,8 +197,8 @@ ZSTD_DUBT_findBetterDictMatch (
|
|
197
197
|
U32 matchIndex = dictMatchIndex + dictIndexDelta;
|
198
198
|
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
|
199
199
|
DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
|
200
|
-
curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr,
|
201
|
-
bestLength = matchLength, *offsetPtr =
|
200
|
+
curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, STORE_OFFSET(curr - matchIndex), dictMatchIndex, matchIndex);
|
201
|
+
bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
|
202
202
|
}
|
203
203
|
if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
|
204
204
|
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
@@ -218,7 +218,7 @@ ZSTD_DUBT_findBetterDictMatch (
|
|
218
218
|
}
|
219
219
|
|
220
220
|
if (bestLength >= MINMATCH) {
|
221
|
-
U32 const mIndex = curr - (
|
221
|
+
U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex;
|
222
222
|
DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
|
223
223
|
curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
|
224
224
|
}
|
@@ -328,7 +328,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
|
328
328
|
if (matchLength > matchEndIdx - matchIndex)
|
329
329
|
matchEndIdx = matchIndex + (U32)matchLength;
|
330
330
|
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
|
331
|
-
bestLength = matchLength, *offsetPtr =
|
331
|
+
bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
|
332
332
|
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
|
333
333
|
if (dictMode == ZSTD_dictMatchState) {
|
334
334
|
nbCompares = 0; /* in addition to avoiding checking any
|
@@ -368,7 +368,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
|
368
368
|
assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */
|
369
369
|
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
|
370
370
|
if (bestLength >= MINMATCH) {
|
371
|
-
U32 const mIndex = curr - (
|
371
|
+
U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex;
|
372
372
|
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
|
373
373
|
curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
|
374
374
|
}
|
@@ -561,7 +561,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
|
|
561
561
|
/* save best solution */
|
562
562
|
if (currentMl > ml) {
|
563
563
|
ml = currentMl;
|
564
|
-
*offsetPtr = curr - (matchIndex + ddsIndexDelta)
|
564
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
|
565
565
|
if (ip+currentMl == iLimit) {
|
566
566
|
/* best possible, avoids read overflow on next attempt */
|
567
567
|
return ml;
|
@@ -598,7 +598,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
|
|
598
598
|
/* save best solution */
|
599
599
|
if (currentMl > ml) {
|
600
600
|
ml = currentMl;
|
601
|
-
*offsetPtr = curr - (matchIndex + ddsIndexDelta)
|
601
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
|
602
602
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
603
603
|
}
|
604
604
|
}
|
@@ -703,7 +703,7 @@ size_t ZSTD_HcFindBestMatch(
|
|
703
703
|
/* save best solution */
|
704
704
|
if (currentMl > ml) {
|
705
705
|
ml = currentMl;
|
706
|
-
*offsetPtr = curr - matchIndex
|
706
|
+
*offsetPtr = STORE_OFFSET(curr - matchIndex);
|
707
707
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
708
708
|
}
|
709
709
|
|
@@ -738,7 +738,8 @@ size_t ZSTD_HcFindBestMatch(
|
|
738
738
|
/* save best solution */
|
739
739
|
if (currentMl > ml) {
|
740
740
|
ml = currentMl;
|
741
|
-
|
741
|
+
assert(curr > matchIndex + dmsIndexDelta);
|
742
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
|
742
743
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
743
744
|
}
|
744
745
|
|
@@ -1244,7 +1245,7 @@ size_t ZSTD_RowFindBestMatch(
|
|
1244
1245
|
/* Save best solution */
|
1245
1246
|
if (currentMl > ml) {
|
1246
1247
|
ml = currentMl;
|
1247
|
-
*offsetPtr = curr - matchIndex
|
1248
|
+
*offsetPtr = STORE_OFFSET(curr - matchIndex);
|
1248
1249
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
1249
1250
|
}
|
1250
1251
|
}
|
@@ -1292,7 +1293,8 @@ size_t ZSTD_RowFindBestMatch(
|
|
1292
1293
|
|
1293
1294
|
if (currentMl > ml) {
|
1294
1295
|
ml = currentMl;
|
1295
|
-
|
1296
|
+
assert(curr > matchIndex + dmsIndexDelta);
|
1297
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
|
1296
1298
|
if (ip+currentMl == iLimit) break;
|
1297
1299
|
}
|
1298
1300
|
}
|
@@ -1537,8 +1539,9 @@ ZSTD_compressBlock_lazy_generic(
|
|
1537
1539
|
#endif
|
1538
1540
|
while (ip < ilimit) {
|
1539
1541
|
size_t matchLength=0;
|
1540
|
-
size_t
|
1542
|
+
size_t offcode=STORE_REPCODE_1;
|
1541
1543
|
const BYTE* start=ip+1;
|
1544
|
+
DEBUGLOG(7, "search baseline (depth 0)");
|
1542
1545
|
|
1543
1546
|
/* check repCode */
|
1544
1547
|
if (isDxS) {
|
@@ -1564,7 +1567,7 @@ ZSTD_compressBlock_lazy_generic(
|
|
1564
1567
|
{ size_t offsetFound = 999999999;
|
1565
1568
|
size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
|
1566
1569
|
if (ml2 > matchLength)
|
1567
|
-
matchLength = ml2, start = ip,
|
1570
|
+
matchLength = ml2, start = ip, offcode=offsetFound;
|
1568
1571
|
}
|
1569
1572
|
|
1570
1573
|
if (matchLength < 4) {
|
@@ -1575,14 +1578,15 @@ ZSTD_compressBlock_lazy_generic(
|
|
1575
1578
|
/* let's try to find a better solution */
|
1576
1579
|
if (depth>=1)
|
1577
1580
|
while (ip<ilimit) {
|
1581
|
+
DEBUGLOG(7, "search depth 1");
|
1578
1582
|
ip ++;
|
1579
1583
|
if ( (dictMode == ZSTD_noDict)
|
1580
|
-
&& (
|
1584
|
+
&& (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
|
1581
1585
|
size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
|
1582
1586
|
int const gain2 = (int)(mlRep * 3);
|
1583
|
-
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)
|
1587
|
+
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1584
1588
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1585
|
-
matchLength = mlRep,
|
1589
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1586
1590
|
}
|
1587
1591
|
if (isDxS) {
|
1588
1592
|
const U32 repIndex = (U32)(ip - base) - offset_1;
|
@@ -1594,30 +1598,31 @@ ZSTD_compressBlock_lazy_generic(
|
|
1594
1598
|
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
1595
1599
|
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
1596
1600
|
int const gain2 = (int)(mlRep * 3);
|
1597
|
-
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)
|
1601
|
+
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1598
1602
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1599
|
-
matchLength = mlRep,
|
1603
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1600
1604
|
}
|
1601
1605
|
}
|
1602
1606
|
{ size_t offset2=999999999;
|
1603
1607
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1604
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1605
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1608
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1609
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4);
|
1606
1610
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1607
|
-
matchLength = ml2,
|
1611
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1608
1612
|
continue; /* search a better one */
|
1609
1613
|
} }
|
1610
1614
|
|
1611
1615
|
/* let's find an even better one */
|
1612
1616
|
if ((depth==2) && (ip<ilimit)) {
|
1617
|
+
DEBUGLOG(7, "search depth 2");
|
1613
1618
|
ip ++;
|
1614
1619
|
if ( (dictMode == ZSTD_noDict)
|
1615
|
-
&& (
|
1620
|
+
&& (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
|
1616
1621
|
size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
|
1617
1622
|
int const gain2 = (int)(mlRep * 4);
|
1618
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1623
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1619
1624
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1620
|
-
matchLength = mlRep,
|
1625
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1621
1626
|
}
|
1622
1627
|
if (isDxS) {
|
1623
1628
|
const U32 repIndex = (U32)(ip - base) - offset_1;
|
@@ -1629,46 +1634,45 @@ ZSTD_compressBlock_lazy_generic(
|
|
1629
1634
|
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
1630
1635
|
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
1631
1636
|
int const gain2 = (int)(mlRep * 4);
|
1632
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1637
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1633
1638
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1634
|
-
matchLength = mlRep,
|
1639
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1635
1640
|
}
|
1636
1641
|
}
|
1637
1642
|
{ size_t offset2=999999999;
|
1638
1643
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1639
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1640
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1644
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1645
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7);
|
1641
1646
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1642
|
-
matchLength = ml2,
|
1647
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1643
1648
|
continue;
|
1644
1649
|
} } }
|
1645
1650
|
break; /* nothing found : store previous solution */
|
1646
1651
|
}
|
1647
1652
|
|
1648
1653
|
/* NOTE:
|
1649
|
-
* start[-
|
1650
|
-
*
|
1651
|
-
* overflows the pointer, which is undefined behavior.
|
1654
|
+
* Pay attention that `start[-value]` can lead to strange undefined behavior
|
1655
|
+
* notably if `value` is unsigned, resulting in a large positive `-value`.
|
1652
1656
|
*/
|
1653
1657
|
/* catch up */
|
1654
|
-
if (
|
1658
|
+
if (STORED_IS_OFFSET(offcode)) {
|
1655
1659
|
if (dictMode == ZSTD_noDict) {
|
1656
|
-
while ( ((start > anchor) & (start - (
|
1657
|
-
&& (start[-1] == (start-(
|
1660
|
+
while ( ((start > anchor) & (start - STORED_OFFSET(offcode) > prefixLowest))
|
1661
|
+
&& (start[-1] == (start-STORED_OFFSET(offcode))[-1]) ) /* only search for offset within prefix */
|
1658
1662
|
{ start--; matchLength++; }
|
1659
1663
|
}
|
1660
1664
|
if (isDxS) {
|
1661
|
-
U32 const matchIndex = (U32)((size_t)(start-base) - (
|
1665
|
+
U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode));
|
1662
1666
|
const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex;
|
1663
1667
|
const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest;
|
1664
1668
|
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
1665
1669
|
}
|
1666
|
-
offset_2 = offset_1; offset_1 = (U32)(
|
1670
|
+
offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode);
|
1667
1671
|
}
|
1668
1672
|
/* store sequence */
|
1669
1673
|
_storeSequence:
|
1670
1674
|
{ size_t const litLength = (size_t)(start - anchor);
|
1671
|
-
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)
|
1675
|
+
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength);
|
1672
1676
|
anchor = ip = start + matchLength;
|
1673
1677
|
}
|
1674
1678
|
|
@@ -1684,8 +1688,8 @@ _storeSequence:
|
|
1684
1688
|
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
|
1685
1689
|
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
|
1686
1690
|
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
|
1687
|
-
|
1688
|
-
ZSTD_storeSeq(seqStore, 0, anchor, iend,
|
1691
|
+
offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset_2 <=> offset_1 */
|
1692
|
+
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
|
1689
1693
|
ip += matchLength;
|
1690
1694
|
anchor = ip;
|
1691
1695
|
continue;
|
@@ -1699,8 +1703,8 @@ _storeSequence:
|
|
1699
1703
|
&& (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
|
1700
1704
|
/* store sequence */
|
1701
1705
|
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
1702
|
-
|
1703
|
-
ZSTD_storeSeq(seqStore, 0, anchor, iend,
|
1706
|
+
offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap repcodes */
|
1707
|
+
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
|
1704
1708
|
ip += matchLength;
|
1705
1709
|
anchor = ip;
|
1706
1710
|
continue; /* faster when present ... (?) */
|
@@ -1901,7 +1905,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1901
1905
|
#endif
|
1902
1906
|
while (ip < ilimit) {
|
1903
1907
|
size_t matchLength=0;
|
1904
|
-
size_t
|
1908
|
+
size_t offcode=STORE_REPCODE_1;
|
1905
1909
|
const BYTE* start=ip+1;
|
1906
1910
|
U32 curr = (U32)(ip-base);
|
1907
1911
|
|
@@ -1923,7 +1927,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1923
1927
|
{ size_t offsetFound = 999999999;
|
1924
1928
|
size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
|
1925
1929
|
if (ml2 > matchLength)
|
1926
|
-
matchLength = ml2, start = ip,
|
1930
|
+
matchLength = ml2, start = ip, offcode=offsetFound;
|
1927
1931
|
}
|
1928
1932
|
|
1929
1933
|
if (matchLength < 4) {
|
@@ -1937,7 +1941,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1937
1941
|
ip ++;
|
1938
1942
|
curr++;
|
1939
1943
|
/* check repCode */
|
1940
|
-
if (
|
1944
|
+
if (offcode) {
|
1941
1945
|
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
|
1942
1946
|
const U32 repIndex = (U32)(curr - offset_1);
|
1943
1947
|
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
@@ -1949,18 +1953,18 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1949
1953
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
1950
1954
|
size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
|
1951
1955
|
int const gain2 = (int)(repLength * 3);
|
1952
|
-
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)
|
1956
|
+
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1953
1957
|
if ((repLength >= 4) && (gain2 > gain1))
|
1954
|
-
matchLength = repLength,
|
1958
|
+
matchLength = repLength, offcode = STORE_REPCODE_1, start = ip;
|
1955
1959
|
} }
|
1956
1960
|
|
1957
1961
|
/* search match, depth 1 */
|
1958
1962
|
{ size_t offset2=999999999;
|
1959
1963
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1960
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1961
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1964
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1965
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4);
|
1962
1966
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1963
|
-
matchLength = ml2,
|
1967
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1964
1968
|
continue; /* search a better one */
|
1965
1969
|
} }
|
1966
1970
|
|
@@ -1969,7 +1973,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1969
1973
|
ip ++;
|
1970
1974
|
curr++;
|
1971
1975
|
/* check repCode */
|
1972
|
-
if (
|
1976
|
+
if (offcode) {
|
1973
1977
|
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
|
1974
1978
|
const U32 repIndex = (U32)(curr - offset_1);
|
1975
1979
|
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
@@ -1981,36 +1985,36 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1981
1985
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
1982
1986
|
size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
|
1983
1987
|
int const gain2 = (int)(repLength * 4);
|
1984
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1988
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1985
1989
|
if ((repLength >= 4) && (gain2 > gain1))
|
1986
|
-
matchLength = repLength,
|
1990
|
+
matchLength = repLength, offcode = STORE_REPCODE_1, start = ip;
|
1987
1991
|
} }
|
1988
1992
|
|
1989
1993
|
/* search match, depth 2 */
|
1990
1994
|
{ size_t offset2=999999999;
|
1991
1995
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1992
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1993
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1996
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1997
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7);
|
1994
1998
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1995
|
-
matchLength = ml2,
|
1999
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1996
2000
|
continue;
|
1997
2001
|
} } }
|
1998
2002
|
break; /* nothing found : store previous solution */
|
1999
2003
|
}
|
2000
2004
|
|
2001
2005
|
/* catch up */
|
2002
|
-
if (
|
2003
|
-
U32 const matchIndex = (U32)((size_t)(start-base) - (
|
2006
|
+
if (STORED_IS_OFFSET(offcode)) {
|
2007
|
+
U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode));
|
2004
2008
|
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
|
2005
2009
|
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
|
2006
2010
|
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
2007
|
-
offset_2 = offset_1; offset_1 = (U32)(
|
2011
|
+
offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode);
|
2008
2012
|
}
|
2009
2013
|
|
2010
2014
|
/* store sequence */
|
2011
2015
|
_storeSequence:
|
2012
2016
|
{ size_t const litLength = (size_t)(start - anchor);
|
2013
|
-
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)
|
2017
|
+
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength);
|
2014
2018
|
anchor = ip = start + matchLength;
|
2015
2019
|
}
|
2016
2020
|
|
@@ -2027,8 +2031,8 @@ _storeSequence:
|
|
2027
2031
|
/* repcode detected we should take it */
|
2028
2032
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
2029
2033
|
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
|
2030
|
-
|
2031
|
-
ZSTD_storeSeq(seqStore, 0, anchor, iend,
|
2034
|
+
offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset history */
|
2035
|
+
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
|
2032
2036
|
ip += matchLength;
|
2033
2037
|
anchor = ip;
|
2034
2038
|
continue; /* faster when present ... (?) */
|
@@ -579,7 +579,9 @@ size_t ZSTD_ldm_generateSequences(
|
|
579
579
|
return 0;
|
580
580
|
}
|
581
581
|
|
582
|
-
void
|
582
|
+
void
|
583
|
+
ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
|
584
|
+
{
|
583
585
|
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
|
584
586
|
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
|
585
587
|
if (srcSize <= seq->litLength) {
|
@@ -709,8 +711,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
|
709
711
|
rep[0] = sequence.offset;
|
710
712
|
/* Store the sequence */
|
711
713
|
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
|
712
|
-
sequence.offset
|
713
|
-
sequence.matchLength
|
714
|
+
STORE_OFFSET(sequence.offset),
|
715
|
+
sequence.matchLength);
|
714
716
|
ip += sequence.matchLength;
|
715
717
|
}
|
716
718
|
}
|